shaw/SillyTavern
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Immutable public and global content management (#5390)

Browse Source 
* Use custom init script instead of postinstall

* Revert changes to start scripts in src\electron

* Add global data to content manager

* Add migration for public overrides and user.css location update

* Update npm publish workflow to use 'omit=dev' flag in npm ci commands

* Rename user.css readme file

* Fix indentation in userCssMiddleware function

* Add directory creation for content target

* Restore template compile location

* Move stylesheet up in index.json

* Use path.resolve for user.css file path in userCssMiddleware

* Correct capitalization in "Not Found" error page title and heading

* Remove init run from startup scripts

* Simplify user CSS file path resolution

* Update userCssMiddleware comment
This commit is contained in:
Cohee
2026-04-05 19:32:28 +03:00
committed by GitHub
parent 9e0ecefd64
commit 8e8f501279
20 changed files with 196 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/npm-publish.yml
+2 -2
View File
@@ -19,7 +19,7 @@ jobs:
- uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
with:
node-version: 24
- run: npm ci
- run: npm ci --omit=dev --ignore-scripts
publish-npm:
needs: build
@@ -30,5 +30,5 @@ jobs:
with:
node-version: 24
registry-url: https://registry.npmjs.org/
- run: npm ci
- run: npm ci --omit=dev --ignore-scripts
- run: npm publish
Start.bat
-1
View File
@@ -2,7 +2,6 @@
pushd %~dp0
set NODE_ENV=production
call npm install --no-save --no-audit --no-fund --loglevel=error --no-progress --omit=dev --ignore-scripts
call npm run init
node server.js %*
pause
popd
UpdateAndStart.bat
-1
View File
@@ -21,7 +21,6 @@ if %errorlevel% neq 0 (
)
set NODE_ENV=production
call npm install --no-save --no-audit --no-fund --loglevel=error --no-progress --omit=dev --ignore-scripts
call npm run init
node server.js %*
:end
pause
UpdateForkAndStart.bat
-1
View File
@@ -103,7 +103,6 @@ if %errorlevel% neq 0 (
echo Installing npm packages and starting server
set NODE_ENV=production
call npm install --no-save --no-audit --no-fund --loglevel=error --no-progress --omit=dev --ignore-scripts
call npm run init
node server.js %*
:end
default/public/error/forbidden-by-whitelist.html → default/content/errors/forbidden-by-whitelist.html
View File
default/public/error/host-not-allowed.html → default/content/errors/host-not-allowed.html
View File
default/public/error/unauthorized.html → default/content/errors/unauthorized.html
View File
default/public/error/url-not-found.html → default/content/errors/url-not-found.html
+2 -2
View File
@@ -2,11 +2,11 @@
<html>
<head>
<title>Not found</title>
<title>Not Found</title>
</head>
<body>
<h1>Not found</h1>
<h1>Not Found</h1>
<p>
The requested URL was not found on this server.
</p>
default/content/index.json
+20
View File
@@ -654,5 +654,25 @@
{
"filename": "presets/context/Gemma 4.json",
"type": "context"
},
{
"filename": "user.css",
"type": "stylesheet"
},
{
"filename": "errors/forbidden-by-whitelist.html",
"type": "error_page"
},
{
"filename": "errors/host-not-allowed.html",
"type": "error_page"
},
{
"filename": "errors/unauthorized.html",
"type": "error_page"
},
{
"filename": "errors/url-not-found.html",
"type": "error_page"
}
]
default/public/css/user.css → default/content/user.css
View File
public/css/!USER-CSS-README.md
+7
View File
@@ -0,0 +1,7 @@
# Looking for user.css?
user.css is now located under your data root directory in the "_css" folder.
Example for the default data root:
/data/_css/user.css
src/endpoints/content-manager.js
+94 -24
View File
@@ -53,8 +53,31 @@ export const CONTENT_TYPES = {
QUICK_REPLIES: 'quick_replies',
SYSPROMPT: 'sysprompt',
REASONING: 'reasoning',
ERROR_PAGE: 'error_page',
STYLESHEET: 'stylesheet',
};
/**
* @enum {string}
*/
export const CONTENT_SCOPE = {
USER: 'user',
GLOBAL: 'global',
};
/**
* Gets the scope of a content type.
* @param {CONTENT_TYPES} type Content type
* @returns {CONTENT_SCOPE} Resolved content scope
*/
function getScopeByType(type) {
const globalTypes = [
CONTENT_TYPES.ERROR_PAGE,
CONTENT_TYPES.STYLESHEET,
];
return globalTypes.includes(type) ? CONTENT_SCOPE.GLOBAL : CONTENT_SCOPE.USER;
}
/**
* Gets the default presets from the content directory.
* @param {import('../users.js').UserDirectoryList} directories User directories
@@ -62,13 +85,13 @@ export const CONTENT_TYPES = {
*/
export function getDefaultPresets(directories) {
try {
const contentIndex = getContentIndex();
const contentIndex = getContentIndex(CONTENT_SCOPE.USER);
const presets = [];
for (const contentItem of contentIndex) {
if (contentItem.type.endsWith('_preset') || ['instruct', 'context', 'sysprompt', 'reasoning'].includes(contentItem.type)) {
contentItem.name = path.parse(contentItem.filename).name;
contentItem.folder = getTargetByType(contentItem.type, directories);
contentItem.folder = getUserTargetByType(contentItem.type, directories);
presets.push(contentItem);
}
}
@@ -102,24 +125,18 @@ export function getDefaultPresetFile(filename) {
}
/**
* Seeds content for a user.
* Seeds content from a content index into a target location.
* @param {ContentItem[]} contentIndex Content index
* @param {import('../users.js').UserDirectoryList} directories User directories
* @param {string[]} forceCategories List of categories to force check (even if content check is skipped)
* @returns {Promise<boolean>} Whether any content was added
* @param {string} contentLogPath Path to the content log file
* @param {(type: string) => string | null} resolveTarget Function to resolve the target directory for a content type
* @param {string[]} [forceCategories] List of categories to force check (even if content check is skipped)
* @returns {boolean} Whether any content was added
*/
async function seedContentForUser(contentIndex, directories, forceCategories) {
function seedContent(contentIndex, contentLogPath, resolveTarget, forceCategories) {
let anyContentAdded = false;
if (!fs.existsSync(directories.root)) {
fs.mkdirSync(directories.root, { recursive: true });
}
const contentLogPath = path.join(directories.root, 'content.log');
const contentLog = getContentLog(contentLogPath);
for (const contentItem of contentIndex) {
// If the content item is already in the log, skip it
if (contentLog.includes(contentItem.filename) && !forceCategories?.includes(contentItem.type)) {
continue;
}
@@ -136,7 +153,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) {
continue;
}
const contentTarget = getTargetByType(contentItem.type, directories);
const contentTarget = resolveTarget(contentItem.type);
if (!contentTarget) {
console.warn(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`);
@@ -152,6 +169,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) {
continue;
}
fs.mkdirSync(contentTarget, { recursive: true });
fs.cpSync(contentPath, targetPath, { recursive: true, force: false });
setPermissionsSync(targetPath);
console.info(`Content file ${contentItem.filename} copied to ${contentTarget}`);
@@ -162,6 +180,32 @@ async function seedContentForUser(contentIndex, directories, forceCategories) {
return anyContentAdded;
}
/**
* Seeds content for a user.
* @param {ContentItem[]} contentIndex Content index
* @param {import('../users.js').UserDirectoryList} directories User directories
* @param {string[]} forceCategories List of categories to force check (even if content check is skipped)
* @returns {Promise<boolean>} Whether any content was added
*/
async function seedContentForUser(contentIndex, directories, forceCategories) {
if (!fs.existsSync(directories.root)) {
fs.mkdirSync(directories.root, { recursive: true });
}
const contentLogPath = path.join(directories.root, 'content.log');
return seedContent(contentIndex, contentLogPath, (type) => getUserTargetByType(type, directories), forceCategories);
}
/**
* Seeds global content that is not user-specific, such as error pages.
* @param {ContentItem[]} contentIndex Content index
* @returns {Promise<boolean>} Whether any content was added
*/
async function seedGlobalContent(contentIndex) {
const contentLogPath = path.join(globalThis.DATA_ROOT, 'content.log');
return seedContent(contentIndex, contentLogPath, getGlobalTargetByType);
}
/**
* Checks for new content and seeds it for all users.
* @param {import('../users.js').UserDirectoryList[]} directoriesList List of user directories
@@ -175,13 +219,19 @@ export async function checkForNewContent(directoriesList, forceCategories = [])
return;
}
const contentIndex = getContentIndex();
const userContentIndex = getContentIndex(CONTENT_SCOPE.USER);
const globalContentIndex = getContentIndex(CONTENT_SCOPE.GLOBAL);
let anyContentAdded = false;
for (const directories of directoriesList) {
const seedResult = await seedContentForUser(contentIndex, directories, forceCategories);
const globalSeedResult = await seedGlobalContent(globalContentIndex);
if (globalSeedResult) {
anyContentAdded = true;
}
if (seedResult) {
for (const directories of directoriesList) {
const userSeedResult = await seedContentForUser(userContentIndex, directories, forceCategories);
if (userSeedResult) {
anyContentAdded = true;
}
}
@@ -198,9 +248,10 @@ export async function checkForNewContent(directoriesList, forceCategories = [])
/**
* Gets combined content index from the content and scaffold directories.
* @param {CONTENT_SCOPE} scope Scope of content to get
* @returns {ContentItem[]} Array of content index
*/
function getContentIndex() {
function getContentIndex(scope = CONTENT_SCOPE.USER) {
const result = [];
if (fs.existsSync(scaffoldIndexPath)) {
@@ -209,6 +260,7 @@ function getContentIndex() {
if (Array.isArray(scaffoldIndex)) {
scaffoldIndex.forEach((item) => {
item.folder = scaffoldDirectory;
item.scope = getScopeByType(item.type);
});
result.push(...scaffoldIndex);
}
@@ -220,22 +272,24 @@ function getContentIndex() {
if (Array.isArray(contentIndex)) {
contentIndex.forEach((item) => {
item.folder = contentDirectory;
item.scope = getScopeByType(item.type);
});
result.push(...contentIndex);
}
}
return result;
return result.filter((item) => item.scope === scope);
}
/**
* Gets content by type and format.
* @param {string} type Type of content
* @param {'json'|'string'|'raw'} format Format of content
* @param {CONTENT_SCOPE} scope Scope of content to get
* @returns {string[]|Buffer[]} Array of content
*/
export function getContentOfType(type, format) {
const contentIndex = getContentIndex();
export function getContentOfType(type, format, scope = CONTENT_SCOPE.USER) {
const contentIndex = getContentIndex(scope);
const indexItems = contentIndex.filter((item) => item.type === type && item.folder);
const files = [];
for (const item of indexItems) {
@@ -269,7 +323,7 @@ export function getContentOfType(type, format) {
* @param {import('../users.js').UserDirectoryList} directories User directories
* @returns {string | null} Target directory
*/
function getTargetByType(type, directories) {
export function getUserTargetByType(type, directories) {
switch (type) {
case CONTENT_TYPES.SETTINGS:
return directories.root;
@@ -312,6 +366,22 @@ function getTargetByType(type, directories) {
}
}
/**
* Gets the target directory for global content types.
* @param {CONTENT_TYPES} type Content type
* @returns {string | null} Target directory
*/
export function getGlobalTargetByType(type) {
switch (type) {
case CONTENT_TYPES.ERROR_PAGE:
return path.join(globalThis.DATA_ROOT, '_errors');
case CONTENT_TYPES.STYLESHEET:
return path.join(globalThis.DATA_ROOT, '_css');
default:
return null;
}
}
/**
* Gets the content log from the content log file.
* @param {string} contentLogPath Path to the content log file
src/middleware/basicAuth.js
+2 -1
View File
@@ -3,6 +3,7 @@
* allow access to the endpoint after successful authentication.
*/
import { Buffer } from 'node:buffer';
import path from 'node:path';
import storage from 'node-persist';
import { getAllUserHandles, toKey, getPasswordHash } from '../users.js';
import { getConfigValue, safeReadFileSync } from '../util.js';
@@ -11,7 +12,7 @@ const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false, 'boolean')
const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false, 'boolean');
const basicAuthMiddleware = async function (request, response, callback) {
const unauthorizedWebpage = safeReadFileSync('./public/error/unauthorized.html') ?? '';
const unauthorizedWebpage = safeReadFileSync(path.join(globalThis.DATA_ROOT, '_errors', 'unauthorized.html')) ?? '';
const unauthorizedResponse = (res) => {
res.set('WWW-Authenticate', 'Basic realm="SillyTavern", charset="UTF-8"');
return res.status(401).send(unauthorizedWebpage);
src/middleware/hostWhitelist.js
+1 -4
View File
@@ -1,6 +1,5 @@
import path from 'node:path';
import { color, getConfigValue, safeReadFileSync } from '../util.js';
import { serverDirectory } from '../server-directory.js';
import { isHostAllowed, hostValidationMiddleware } from 'host-validation-middleware';
const knownHosts = new Set();
@@ -10,11 +9,9 @@ const hostWhitelistEnabled = !!getConfigValue('hostWhitelist.enabled', false);
const hostWhitelist = Object.freeze(getConfigValue('hostWhitelist.hosts', []));
const hostWhitelistScan = !!getConfigValue('hostWhitelist.scan', false, 'boolean');
const hostNotAllowedHtml = safeReadFileSync(path.join(serverDirectory, 'public/error/host-not-allowed.html'))?.toString() ?? '';
const validationMiddleware = hostValidationMiddleware({
allowedHosts: hostWhitelist,
generateErrorMessage: () => hostNotAllowedHtml,
generateErrorMessage: () => safeReadFileSync(path.join(globalThis.DATA_ROOT, '_errors', 'host-not-allowed.html'))?.toString() ?? '',
errorResponseContentType: 'text/html',
});
src/middleware/userCss.js
+19
View File
@@ -0,0 +1,19 @@
import path from 'node:path';
import fs from 'node:fs';
/**
* Provides an Express middleware function that serves a user-defined CSS file from the data directory if it exists.
* @type {import('express').Handler}
*/
export function userCssMiddleware(req, res, next) {
if (req.method === 'GET' && req.path === '/css/user.css') {
const userCssPath = path.resolve(path.join(globalThis.DATA_ROOT, '_css', 'user.css'));
if (fs.existsSync(userCssPath)) {
res.sendFile(userCssPath);
return;
}
}
next();
}
export default userCssMiddleware;
src/middleware/whitelist.js
+1 -1
View File
@@ -100,7 +100,7 @@ async function addDockerHostsToWhitelist() {
*/
export default async function getWhitelistMiddleware() {
const forbiddenWebpage = Handlebars.compile(
safeReadFileSync('./public/error/forbidden-by-whitelist.html') ?? '',
safeReadFileSync(path.join(globalThis.DATA_ROOT, '_errors', 'forbidden-by-whitelist.html')) ?? '',
);
const noLogPaths = [
src/server-init.js
-102
View File
@@ -1,113 +1,11 @@
/**
* Scripts to be done before starting the server for the first time.
*/
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import yaml from 'yaml';
import chalk from 'chalk';
import { createRequire } from 'node:module';
import { addMissingConfigValues } from './config-init.js';
/**
* Colorizes console output.
*/
const color = chalk;
/**
* Converts the old config.conf file to the new config.yaml format.
*/
function convertConfig() {
if (fs.existsSync('./config.conf')) {
if (fs.existsSync('./config.yaml')) {
console.log(color.yellow('Both config.conf and config.yaml exist. Please delete config.conf manually.'));
return;
}
try {
console.log(color.blue('Converting config.conf to config.yaml. Your old config.conf will be renamed to config.conf.bak'));
fs.renameSync('./config.conf', './config.conf.cjs'); // Force loading as CommonJS
const require = createRequire(import.meta.url);
const config = require(path.join(process.cwd(), './config.conf.cjs'));
fs.copyFileSync('./config.conf.cjs', './config.conf.bak');
fs.rmSync('./config.conf.cjs');
fs.writeFileSync('./config.yaml', yaml.stringify(config));
console.log(color.green('Conversion successful. Please check your config.yaml and fix it if necessary.'));
} catch (error) {
console.error(color.red('FATAL: Config conversion failed. Please check your config.conf file and try again.'), error);
return;
}
}
}
/**
* Creates the default config files if they don't exist yet.
*/
function createDefaultFiles() {
/**
* @typedef DefaultItem
* @type {object}
* @property {'file' | 'directory'} type - Whether the item should be copied as a single file or merged into a directory structure.
* @property {string} defaultPath - The path to the default item (typically in `default/`).
* @property {string} productionPath - The path to the copied item for production use.
*/
/** @type {DefaultItem[]} */
const defaultItems = [
{
type: 'file',
defaultPath: './default/config.yaml',
productionPath: './config.yaml',
},
{
type: 'directory',
defaultPath: './default/public/',
productionPath: './public/',
},
];
for (const defaultItem of defaultItems) {
try {
if (defaultItem.type === 'file') {
if (!fs.existsSync(defaultItem.productionPath)) {
fs.copyFileSync(
defaultItem.defaultPath,
defaultItem.productionPath,
);
console.log(
color.green(`Created default file: ${defaultItem.productionPath}`),
);
}
} else if (defaultItem.type === 'directory') {
fs.cpSync(defaultItem.defaultPath, defaultItem.productionPath, {
force: false, // Don't overwrite existing files!
recursive: true,
});
console.log(
color.green(`Synchronized missing files: ${defaultItem.productionPath}`),
);
} else {
throw new Error(
'FATAL: Unexpected default file format in `server-init.js#createDefaultFiles()`.',
);
}
} catch (error) {
console.error(
color.red(
`FATAL: Could not write default ${defaultItem.type}: ${defaultItem.productionPath}`,
),
error,
);
}
}
}
try {
// 0. Convert config.conf to config.yaml
convertConfig();
// 1. Create default config files
createDefaultFiles();
// 2. Add missing config values
addMissingConfigValues(path.join(process.cwd(), './config.yaml'));
} catch (error) {
console.error(error);
src/server-main.js
+5 -1
View File
@@ -37,6 +37,7 @@ import {
getSessionCookieAge,
verifySecuritySettings,
loginPageMiddleware,
migratePublicOverrides,
} from './users.js';
import getWebpackServeMiddleware from './middleware/webpack-serve.js';
@@ -48,6 +49,7 @@ import initRequestProxy from './request-proxy.js';
import cacheBuster from './middleware/cacheBuster.js';
import corsProxyMiddleware from './middleware/corsProxy.js';
import hostWhitelistMiddleware from './middleware/hostWhitelist.js';
import userCssMiddleware from './middleware/userCss.js';
import {
getVersion,
color,
@@ -229,6 +231,7 @@ app.get('/login', loginPageMiddleware);
// Host frontend assets
const webpackMiddleware = getWebpackServeMiddleware();
app.use(webpackMiddleware);
app.use(userCssMiddleware);
app.use(express.static(path.join(serverDirectory, 'public'), {}));
// Public API
@@ -430,7 +433,7 @@ async function postSetupTasks(result) {
* Registers a not-found error response if a not-found error page exists. Should only be called after all other middlewares have been registered.
*/
function apply404Middleware() {
const notFoundWebpage = safeReadFileSync(path.join(serverDirectory, 'public/error/url-not-found.html')) ?? '';
const notFoundWebpage = safeReadFileSync(path.join(globalThis.DATA_ROOT, '_errors', 'url-not-found.html')) ?? '';
app.use((req, res) => {
res.status(404).send(notFoundWebpage);
});
@@ -459,6 +462,7 @@ initUserStorage(globalThis.DATA_ROOT)
.then(ensurePublicDirectoriesExist)
.then(migrateUserData)
.then(migrateSystemPrompts)
.then(migratePublicOverrides)
.then(verifySecuritySettings)
.then(preSetupTasks)
.then(apply404Middleware)
src/users.js
+43 -1
View File
@@ -16,7 +16,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
import sanitize from 'sanitize-filename';
import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE, UPLOADS_DIRECTORY } from './constants.js';
import { getConfigValue, color, delay, generateTimestamp, invalidateFirefoxCache, isPathUnderParent } from './util.js';
import { getConfigValue, color, delay, generateTimestamp, invalidateFirefoxCache, isPathUnderParent, setPermissionsSync } from './util.js';
import { allowKeysExposure, readSecret, writeSecret, SECRETS_FILE } from './endpoints/secrets.js';
import { getContentOfType } from './endpoints/content-manager.js';
import { serverDirectory } from './server-directory.js';
@@ -484,6 +484,48 @@ export async function migrateSystemPrompts() {
}
}
export async function migratePublicOverrides() {
const migrationMap = [
{
oldPath: path.join(serverDirectory, 'public', 'error', 'forbidden-by-whitelist.html'),
newPath: path.join(globalThis.DATA_ROOT, '_errors', 'forbidden-by-whitelist.html'),
},
{
oldPath: path.join(serverDirectory, 'public', 'error', 'host-not-allowed.html'),
newPath: path.join(globalThis.DATA_ROOT, '_errors', 'host-not-allowed.html'),
},
{
oldPath: path.join(serverDirectory, 'public', 'error', 'unauthorized.html'),
newPath: path.join(globalThis.DATA_ROOT, '_errors', 'unauthorized.html'),
},
{
oldPath: path.join(serverDirectory, 'public', 'error', 'url-not-found.html'),
newPath: path.join(globalThis.DATA_ROOT, '_errors', 'url-not-found.html'),
},
{
oldPath: path.join(serverDirectory, 'public', 'css', 'user.css'),
newPath: path.join(globalThis.DATA_ROOT, '_css', 'user.css'),
},
];
for (const { oldPath, newPath } of migrationMap) {
try {
if (fs.existsSync(newPath)) {
continue;
}
if (fs.existsSync(oldPath)) {
fs.mkdirSync(path.dirname(newPath), { recursive: true });
fs.cpSync(oldPath, newPath, { force: true });
fs.unlinkSync(oldPath);
setPermissionsSync(newPath);
console.log(`Migrated ${path.basename(oldPath)} to data root.`);
}
} catch (error) {
console.error(`Error migrating ${oldPath} to ${newPath}:`, error);
}
}
}
/**
* Converts a user handle to a storage key.
* @param {string} handle User handle
start.sh
-1
View File
@@ -11,7 +11,6 @@ fi
echo "Installing Node Modules..."
export NODE_ENV=production
npm install --no-save --no-audit --no-fund --loglevel=error --no-progress --omit=dev --ignore-scripts
npm run init
echo "Entering SillyTavern..."
node "server.js" "$@"