Bulk extension field updates via merge-attributes with UNSET_VALUE sentinel (#5471)
* feat: add bulk extension field updates with UNSET_VALUE sentinel for key deletion - Add `UNSET_VALUE` sentinel constant to signal complete field removal from character cards - Add `writeExtensionFieldBulk()` function to update extension fields across multiple characters in a single API call - Add `deleteValueByPath()` utility function to remove nested object keys by dot-path - Update `writeExtensionField()` to support `UNSET_VALUE` for deleting extension keys - Extend `/api/characters/merge-attributes * Revert package-lock.json changes * Allow null values in merge-attributes filter path validation Change filter.path existence check to only skip on undefined, not null. This allows merging attributes when the existing value is explicitly null, treating null as a valid value rather than absence of a value. * fix: share forbiddenRegExp between modules * feat: add writeExtensionFieldBulk and UNSET_VALUE constant to getContext * Update src/endpoints/characters.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: validate for .png extension * Update public/scripts/extensions.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: extract shouldSkip logic as a function param to avoid double parsing --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wolfsblvt
@@ -3,7 +3,7 @@ import { DOMPurify, Popper } from '../lib.js';
|
||||
import { eventSource, event_types, saveSettings, saveSettingsDebounced, getRequestHeaders, animation_duration, CLIENT_VERSION } from '../script.js';
|
||||
import { POPUP_RESULT, POPUP_TYPE, Popup } from './popup.js';
|
||||
import { renderTemplate, renderTemplateAsync } from './templates.js';
|
||||
import { delay, equalsIgnoreCaseAndAccents, isSubsetOf, sanitizeSelector, setValueByPath, versionCompare } from './utils.js';
|
||||
import { delay, deleteValueByPath, equalsIgnoreCaseAndAccents, isSubsetOf, sanitizeSelector, setValueByPath, versionCompare } from './utils.js';
|
||||
import { getContext } from './st-context.js';
|
||||
import { isAdmin } from './user.js';
|
||||
import { addLocaleData, getCurrentLocale, t } from './i18n.js';
|
||||
@@ -1839,6 +1839,18 @@ export async function runGenerationInterceptors(chat, contextSize, type) {
|
||||
return aborted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentinel value that signals a field should be completely removed (unset)
|
||||
* from the character card rather than being set to any value. Pass this as
|
||||
* the `value` argument to {@link writeExtensionField} or
|
||||
* {@link writeExtensionFieldBulk} to delete the key entirely.
|
||||
*
|
||||
* Using `null` as a value will set the field to `null` (the key remains).
|
||||
* Using this sentinel will delete the key from the character card.
|
||||
* @type {string}
|
||||
*/
|
||||
export const UNSET_VALUE = '__@@UNSET@@__';
|
||||
|
||||
/**
|
||||
* Writes a field to the character's data extensions object.
|
||||
* @param {number|string} characterId Index in the character array
|
||||
@@ -1853,13 +1865,23 @@ export async function writeExtensionField(characterId, key, value) {
|
||||
console.warn('Character not found', characterId);
|
||||
return;
|
||||
}
|
||||
const path = `data.extensions.${key}`;
|
||||
setValueByPath(character, path, value);
|
||||
const extensionPath = `data.extensions.${key}`;
|
||||
const isUnset = value === UNSET_VALUE;
|
||||
|
||||
if (isUnset) {
|
||||
deleteValueByPath(character, extensionPath);
|
||||
} else {
|
||||
setValueByPath(character, extensionPath, value);
|
||||
}
|
||||
|
||||
// Process JSON data
|
||||
if (character.json_data) {
|
||||
const jsonData = JSON.parse(character.json_data);
|
||||
setValueByPath(jsonData, path, value);
|
||||
if (isUnset) {
|
||||
deleteValueByPath(jsonData, extensionPath);
|
||||
} else {
|
||||
setValueByPath(jsonData, extensionPath, value);
|
||||
}
|
||||
character.json_data = JSON.stringify(jsonData);
|
||||
|
||||
// Make sure the data doesn't get lost when saving the current character
|
||||
@@ -1888,6 +1910,107 @@ export async function writeExtensionField(characterId, key, value) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {object} BulkExtensionFieldResult
|
||||
* @property {string[]} updated Avatar filenames that were successfully updated
|
||||
* @property {string[]} skipped Avatar filenames skipped (filter didn't match or unreadable)
|
||||
* @property {string[]} failed Avatar filenames where the update failed
|
||||
*/
|
||||
|
||||
/**
|
||||
* Writes (or deletes) an extension field for multiple characters in a single
|
||||
* bulk request. Unlike {@link writeExtensionField}, this sends one API call
|
||||
* for all characters, and the server processes them in parallel.
|
||||
*
|
||||
* When `value` is {@link UNSET_VALUE} the extension key is **deleted** from
|
||||
* each matching character card. Passing `null` sets the field to `null`
|
||||
* (the key is preserved).
|
||||
*
|
||||
* @param {string[]|null} avatars Avatar filenames to update. Pass `null` or an
|
||||
* empty array to target **all** characters in the user's character directory.
|
||||
* @param {string} key Extension field name (e.g. "greeting_tools")
|
||||
* @param {any} value Field value, `null` to set null, or
|
||||
* {@link UNSET_VALUE} to delete the key entirely
|
||||
* @param {object} [options={}] Optional settings
|
||||
* @param {string} [options.filterPath] Dot-path filter — the server will only
|
||||
* update characters where this path is present and not `undefined`;
|
||||
* `null` still counts as a match. Useful when the frontend has shallow
|
||||
* character data and cannot pre-filter.
|
||||
* Defaults to `data.extensions.<key>` when unsetting, so deletion requests
|
||||
* automatically skip characters where the field is missing/`undefined`.
|
||||
* @returns {Promise<BulkExtensionFieldResult>} Summary of the bulk operation
|
||||
*/
|
||||
export async function writeExtensionFieldBulk(avatars, key, value, { filterPath } = {}) {
|
||||
const context = getContext();
|
||||
const extensionPath = `data.extensions.${key}`;
|
||||
const isUnset = value === UNSET_VALUE;
|
||||
|
||||
// Build the server request
|
||||
const requestBody = {
|
||||
avatars: Array.isArray(avatars) && avatars.length > 0 ? avatars : [],
|
||||
data: {
|
||||
data: {
|
||||
extensions: {
|
||||
[key]: value,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Default filter: when unsetting, only touch characters that have the field
|
||||
const resolvedFilterPath = filterPath ?? (isUnset ? extensionPath : undefined);
|
||||
if (resolvedFilterPath) {
|
||||
requestBody.filter = { path: resolvedFilterPath };
|
||||
}
|
||||
|
||||
const mergeResponse = await fetch('/api/characters/merge-attributes', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
if (!mergeResponse.ok) {
|
||||
console.error('Bulk extension field update failed', mergeResponse.statusText);
|
||||
return { updated: [], skipped: [], failed: [] };
|
||||
}
|
||||
|
||||
/** @type {BulkExtensionFieldResult} */
|
||||
const result = await mergeResponse.json();
|
||||
|
||||
// Sync in-memory character objects for successfully updated characters
|
||||
const updatedSet = new Set(result.updated);
|
||||
for (const character of context.characters) {
|
||||
if (!character || !updatedSet.has(character.avatar)) continue;
|
||||
|
||||
if (isUnset) {
|
||||
deleteValueByPath(character, extensionPath);
|
||||
} else {
|
||||
setValueByPath(character, extensionPath, value);
|
||||
}
|
||||
|
||||
// Keep json_data in sync
|
||||
if (character.json_data) {
|
||||
const jsonData = JSON.parse(character.json_data);
|
||||
if (isUnset) {
|
||||
deleteValueByPath(jsonData, extensionPath);
|
||||
} else {
|
||||
setValueByPath(jsonData, extensionPath, value);
|
||||
}
|
||||
character.json_data = JSON.stringify(jsonData);
|
||||
}
|
||||
}
|
||||
|
||||
// If the currently active character was updated, sync the hidden input
|
||||
if (context.characterId !== undefined) {
|
||||
const activeChar = context.characters[context.characterId];
|
||||
if (activeChar && updatedSet.has(activeChar.avatar) && activeChar.json_data) {
|
||||
$('#character_json_data').val(activeChar.json_data);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompts the user to enter the Git URL of the extension to import.
|
||||
* After obtaining the Git URL, makes a POST request to '/api/extensions/install' to import the extension.
|
||||
|
||||
@@ -77,7 +77,9 @@ import {
|
||||
renderExtensionTemplate,
|
||||
renderExtensionTemplateAsync,
|
||||
saveMetadataDebounced,
|
||||
UNSET_VALUE,
|
||||
writeExtensionField,
|
||||
writeExtensionFieldBulk,
|
||||
} from './extensions.js';
|
||||
import { groups, openGroupChat, selected_group, unshallowGroupMembers } from './group-chats.js';
|
||||
import { addLocaleData, getCurrentLocale, t, translate } from './i18n.js';
|
||||
@@ -202,6 +204,7 @@ export function getContext() {
|
||||
generateRaw,
|
||||
generateRawData,
|
||||
writeExtensionField,
|
||||
writeExtensionFieldBulk,
|
||||
getThumbnailUrl,
|
||||
selectCharacterById,
|
||||
messageFormatting,
|
||||
@@ -296,6 +299,9 @@ export function getContext() {
|
||||
symbols: {
|
||||
ignore: IGNORE_SYMBOL,
|
||||
},
|
||||
constants: {
|
||||
unset: UNSET_VALUE,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -2061,6 +2061,23 @@ export function setValueByPath(obj, path, value) {
|
||||
currentObject[keyParts[keyParts.length - 1]] = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a value from a nested object at the given dot-separated path.
|
||||
* @param {object} obj Object to delete from
|
||||
* @param {string} path Dot-separated key path (e.g. "data.extensions.myKey")
|
||||
*/
|
||||
export function deleteValueByPath(obj, path) {
|
||||
const keyParts = path.split('.');
|
||||
let current = obj;
|
||||
for (let i = 0; i < keyParts.length - 1; i++) {
|
||||
if (!current || typeof current !== 'object') return;
|
||||
current = current[keyParts[i]];
|
||||
}
|
||||
if (current && typeof current === 'object') {
|
||||
delete current[keyParts[keyParts.length - 1]];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flashes the given HTML element via CSS flash animation for a defined period
|
||||
* @param {JQuery<HTMLElement>} element - The element to flash
|
||||
|
||||
+164
-28
@@ -13,7 +13,7 @@ import { Jimp, JimpMime } from '../jimp.js';
|
||||
import storage from 'node-persist';
|
||||
|
||||
import { AVATAR_WIDTH, AVATAR_HEIGHT, DEFAULT_AVATAR_PATH } from '../constants.js';
|
||||
import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction } from '../middleware/validateFileName.js';
|
||||
import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction, forbiddenRegExp } from '../middleware/validateFileName.js';
|
||||
import { deepMerge, humanizedDateTime, tryParse, MemoryLimitedMap, getConfigValue, mutateJsonString, clientRelativePath, getUniqueName, sanitizeSafeCharacterReplacements } from '../util.js';
|
||||
import { TavernCardValidator } from '../validator/TavernCardValidator.js';
|
||||
import { parse, read, write } from '../character-card-parser.js';
|
||||
@@ -1219,46 +1219,182 @@ router.post('/edit-attribute', validateAvatarUrlMiddleware, async function (requ
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Sentinel value that signals a field should be completely removed (unset)
|
||||
* from the character card rather than being set to any value. Use this in
|
||||
* the merge payload wherever a key should be deleted.
|
||||
*
|
||||
* Both the server and the frontend share this constant so that callers can
|
||||
* explicitly opt into deletion without overloading `null`.
|
||||
* @type {string}
|
||||
*/
|
||||
const UNSET_SENTINEL = '__@@UNSET@@__';
|
||||
|
||||
/** Maximum number of characters processed in parallel during bulk merge */
|
||||
const BULK_MERGE_CONCURRENCY = 10;
|
||||
|
||||
/**
|
||||
* Recursively walks `source` and removes any key from `target` whose
|
||||
* corresponding value in `source` equals the {@link UNSET_SENTINEL}.
|
||||
* Called after {@link deepMerge} so that the sentinel gets replaced by
|
||||
* an actual key deletion.
|
||||
* @param {object} target The merged character object to clean up
|
||||
* @param {object} source The original update payload (pre-merge clone)
|
||||
*/
|
||||
function processUnsetSentinels(target, source) {
|
||||
for (const key of Object.keys(source)) {
|
||||
if (source[key] === UNSET_SENTINEL) {
|
||||
_.unset(target, key);
|
||||
} else if (_.isPlainObject(source[key]) && _.isPlainObject(target[key])) {
|
||||
processUnsetSentinels(target[key], source[key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a character card, applies a merge update (with sentinel-based
|
||||
* unsetting), validates the result, and writes it back.
|
||||
* @param {string} avatarPath Full path to the character PNG
|
||||
* @param {string} avatar Avatar filename (e.g. "char.png")
|
||||
* @param {object} updateData The merge payload to apply
|
||||
* @param {import("express").Request} request Express request object
|
||||
* @param {((data: any) => boolean) | null} [shouldSkip] Optional function to determine if a character should be skipped based on its original data (used for bulk merge filtering)
|
||||
* @returns {Promise<{ok: boolean, error?: string, skipped?: boolean}>} Result of the merge operation, including any validation error
|
||||
*/
|
||||
async function mergeCharacterUpdate(avatarPath, avatar, updateData, request, shouldSkip = null) {
|
||||
const pngStringData = await readCharacterData(avatarPath);
|
||||
if (!pngStringData) {
|
||||
return { ok: false, error: 'Invalid character file' };
|
||||
}
|
||||
|
||||
let character = JSON.parse(pngStringData);
|
||||
|
||||
if (typeof shouldSkip === 'function' && shouldSkip(character)) {
|
||||
return { ok: false, skipped: true };
|
||||
}
|
||||
|
||||
const update = _.cloneDeep(updateData);
|
||||
_.unset(update, 'json_data');
|
||||
_.unset(character, 'json_data');
|
||||
|
||||
character = deepMerge(character, update);
|
||||
processUnsetSentinels(character, update);
|
||||
|
||||
const validator = new TavernCardValidator(character);
|
||||
//Accept either V1 or V2.
|
||||
if (!validator.validate()) {
|
||||
return { ok: false, error: validator.lastValidationError ?? 'Validation failed' };
|
||||
}
|
||||
|
||||
const targetImg = avatar.replace('.png', '');
|
||||
await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a POST request to edit character properties.
|
||||
*
|
||||
* Merges the request body with the selected character and
|
||||
* validates the result against TavernCard V2 specification.
|
||||
* Operates in two modes depending on the request body:
|
||||
*
|
||||
* @param {Object} request - The HTTP request object.
|
||||
* @param {Object} response - The HTTP response object.
|
||||
* **Single mode** (default behavior) — when `avatar` (string) is present:
|
||||
* Merges the request body with the selected character and validates the
|
||||
* result against TavernCard V2 specification.
|
||||
*
|
||||
* **Bulk mode** — when `avatars` (array) is present:
|
||||
* Applies the same merge to multiple characters in parallel. Supports:
|
||||
* - An explicit list of avatars, or all characters when the array is empty
|
||||
* - An optional server-side `filter` so only characters where a given
|
||||
* JSON path exists and is non-null are updated
|
||||
*
|
||||
* In both modes, any value equal to the sentinel `__@@UNSET@@__` will cause
|
||||
* that key to be **deleted** from the character card instead of being set.
|
||||
*
|
||||
* @param {import("express").Request} request - The HTTP request object
|
||||
* @param {import("express").Response} response - The HTTP response object
|
||||
* @returns {void}
|
||||
* */
|
||||
*/
|
||||
router.post('/merge-attributes', getFileNameValidationFunction('avatar'), async function (request, response) {
|
||||
try {
|
||||
// ── Bulk mode: avatars array is present ──────────────────
|
||||
if (Array.isArray(request.body.avatars)) {
|
||||
const { avatars, data, filter } = request.body;
|
||||
|
||||
if (!_.isPlainObject(data)) {
|
||||
return response.status(400).send({ message: 'No valid update data provided.' });
|
||||
}
|
||||
|
||||
// Determine which avatar files to process
|
||||
let targetAvatars;
|
||||
if (avatars.length > 0) {
|
||||
for (const avatar of avatars) {
|
||||
if (typeof avatar !== 'string' || forbiddenRegExp.test(avatar) || path.extname(avatar).toLowerCase() !== '.png') {
|
||||
return response.status(400).send({ message: `Invalid avatar filename: ${avatar}` });
|
||||
}
|
||||
}
|
||||
targetAvatars = avatars;
|
||||
} else {
|
||||
// Empty array → scan all characters in the directory
|
||||
const files = fs.readdirSync(request.user.directories.characters);
|
||||
targetAvatars = files.filter(file => path.extname(file).toLowerCase() === '.png');
|
||||
}
|
||||
|
||||
const updated = [];
|
||||
const skipped = [];
|
||||
const failed = [];
|
||||
|
||||
/**
|
||||
* Process a single character in bulk: read, filter, merge, validate, write.
|
||||
* @param {string} avatar Avatar filename
|
||||
*/
|
||||
const processOne = async (avatar) => {
|
||||
const avatarPath = path.join(request.user.directories.characters, avatar);
|
||||
|
||||
try {
|
||||
/** @type {(character: object) => boolean} */
|
||||
let shouldSkip = () => false;
|
||||
|
||||
// Apply optional server-side filter before updating the card
|
||||
if (filter && typeof filter.path === 'string') {
|
||||
shouldSkip = (character) => {
|
||||
const value = _.get(character, filter.path);
|
||||
return value === undefined;
|
||||
};
|
||||
}
|
||||
|
||||
const result = await mergeCharacterUpdate(avatarPath, avatar, data, request, shouldSkip);
|
||||
if (result.ok) {
|
||||
updated.push(avatar);
|
||||
} else if (result.skipped) {
|
||||
skipped.push(avatar);
|
||||
} else {
|
||||
console.warn(`Bulk merge failed for ${avatar}:`, result.error);
|
||||
failed.push(avatar);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Bulk merge failed for ${avatar}:`, error);
|
||||
failed.push(avatar);
|
||||
}
|
||||
};
|
||||
|
||||
// Process in parallel with a concurrency limit
|
||||
for (let i = 0; i < targetAvatars.length; i += BULK_MERGE_CONCURRENCY) {
|
||||
const batch = targetAvatars.slice(i, i + BULK_MERGE_CONCURRENCY);
|
||||
await Promise.allSettled(batch.map(processOne));
|
||||
}
|
||||
|
||||
return response.send({ updated, skipped, failed });
|
||||
}
|
||||
|
||||
// ── Single mode (default behavior) ───────────────────────
|
||||
const update = request.body;
|
||||
const avatarPath = path.join(request.user.directories.characters, update.avatar);
|
||||
|
||||
const pngStringData = await readCharacterData(avatarPath);
|
||||
|
||||
if (!pngStringData) {
|
||||
console.error('Error: invalid character file.');
|
||||
return response.status(400).send('Error: invalid character file.');
|
||||
}
|
||||
|
||||
let character = JSON.parse(pngStringData);
|
||||
|
||||
_.unset(update, 'json_data');
|
||||
_.unset(character, 'json_data');
|
||||
|
||||
character = deepMerge(character, update);
|
||||
|
||||
const validator = new TavernCardValidator(character);
|
||||
const targetImg = (update.avatar).replace('.png', '');
|
||||
|
||||
//Accept either V1 or V2.
|
||||
if (validator.validate()) {
|
||||
await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
|
||||
const result = await mergeCharacterUpdate(avatarPath, update.avatar, update, request);
|
||||
if (result.ok) {
|
||||
response.sendStatus(200);
|
||||
} else {
|
||||
console.warn(validator.lastValidationError);
|
||||
response.status(400).send({ message: `Validation failed for ${character.name}`, error: validator.lastValidationError });
|
||||
console.warn(result.error);
|
||||
response.status(400).send({ message: `Validation failed for ${update.avatar}`, error: result.error });
|
||||
}
|
||||
} catch (exception) {
|
||||
response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() });
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import path from 'node:path';
|
||||
|
||||
export const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
|
||||
|
||||
/**
|
||||
* Checks if an object has a toString method.
|
||||
* @param {object} o Object to check
|
||||
@@ -23,7 +25,6 @@ export function getFileNameValidationFunction(fieldName) {
|
||||
*/
|
||||
return function validateAvatarUrlMiddleware(req, res, next) {
|
||||
if (req.body && fieldName in req.body && (typeof req.body[fieldName] === 'string' || hasToString(req.body[fieldName]))) {
|
||||
const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
|
||||
if (forbiddenRegExp.test(req.body[fieldName])) {
|
||||
console.error('An error occurred while validating the request body', {
|
||||
handle: req.user.profile.handle,
|
||||
|
||||
Reference in New Issue
Block a user