shaw/SillyTavern
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Update group chat metadata format (#4805)

Browse Source 
* Migrate group metadata to group chat files

* Skip migration if chat already has metadata

* Fix active group not being set on group conversion

* Improve types in createGroup

* Fix padding in hotswap group avatars

* Fix centering of empty hotswap avatar

* Added automatic backups of migrated data

* Fix 'OVERWRITE' for GC

* Fix metadata parsing order in migration

* Remove color accents from regular migration logs

* Always set gen_id in converted message

* Clone messages before conversion

* Reduce size of add/remove buttons

* Fix group chat file size calculation
This commit is contained in:
Cohee
2025-11-25 23:48:43 +02:00
committed by GitHub
parent 051b275795
commit 929d377da8
17 changed files with 566 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/css/rm-groups.css
+8
View File
@@ -212,6 +212,14 @@
background-color: var(--white30a);
}
.group_select.avatar {
padding: 0;
}
.group_select.missing-avatar.inline_avatar {
justify-content: center;
}
.group_select .avatar {
flex: 0;
}
public/global.d.ts
Vendored
+37
View File
@@ -15,6 +15,42 @@ declare global {
type ChatCompletionSettings = typeof oai_settings;
type TextCompletionSettings = typeof textgenerationwebui_settings;
type MessageTimestamp = string | number | Date;
type Character = import('./scripts/char-data').v1CharData;
interface Group {
id: string;
name: string;
members: string[];
disabled_members: string[];
chat_id: string;
chats: string[];
generation_mode?: number;
generation_mode_join_prefix?: string;
generation_mode_join_suffix?: string;
activation_strategy?: number;
auto_mode_delay?: number;
allow_self_responses?: boolean;
avatar_url?: string;
hideMutedSprites?: boolean;
fav?: boolean;
}
interface ChatFile extends Array<ChatMessage> {
[index: number]: ChatMessage;
0?: ChatHeader;
}
interface ChatHeader {
chat_metadata: ChatMetadata;
}
interface ChatMetadata {
tainted?: boolean;
integrity?: string;
scenario?: string;
persona?: string;
[key: string]: any;
}
interface ChatMessage {
name?: string;
@@ -34,6 +70,7 @@ declare global {
};
interface ChatMessageExtra {
gen_id?: number;
bias?: string;
uses_system_ui?: boolean;
memory?: string;
public/index.html
+2 -3
View File
@@ -7215,9 +7215,8 @@
<div title="Move down" data-action="down" class="right_menu_button fa-solid fa-chevron-down" data-i18n="[title]Move down"></div>
</div>
<div title="View character card" data-action="view" class="right_menu_button fa-solid fa-xl fa-image-portrait" data-i18n="[title]View character card"></div>
<div title="Remove from group" data-action="remove" class="right_menu_button fa-solid fa-2xl fa-xmark" data-i18n="[title]Remove from group">
</div>
<div title="Add to group" data-action="add" class="right_menu_button fa-solid fa-2xl fa-plus" data-i18n="[title]Add to group"></div>
<div title="Remove from group" data-action="remove" class="right_menu_button fa-solid fa-xl fa-xmark" data-i18n="[title]Remove from group"></div>
<div title="Add to group" data-action="add" class="right_menu_button fa-solid fa-xl fa-plus" data-i18n="[title]Add to group"></div>
</div>
</div>
</div>
public/script.js
+18 -33
View File
@@ -65,7 +65,6 @@ import {
renameGroupMember,
createNewGroupChat,
getGroupAvatar,
editGroup,
deleteGroupChat,
renameGroupChat,
importGroupChat,
@@ -377,14 +376,13 @@ export let isSwipingAllowed = true; //false when a swipe is in progress, or swip
let chatSaveTimeout;
let importFlashTimeout;
export let isChatSaving = false;
let chat_create_date = '';
let firstRun = false;
let settingsReady = false;
let currentVersion = '0.0.0';
export let displayVersion = 'SillyTavern';
let generation_started = new Date();
/** @type {import('./scripts/char-data.js').v1CharData[]} */
/** @type {Character[]} */
export let characters = [];
/**
* Stringified index of a currently chosen entity in the characters array.
@@ -411,6 +409,7 @@ export const chatElement = $('#chat');
let dialogueResolve = null;
let dialogueCloseStop = false;
/** @type {ChatMetadata} */
export let chat_metadata = {};
/** @type {StreamingProcessor} */
export let streamingProcessor = null;
@@ -1295,7 +1294,7 @@ export async function deleteCharacterChatByName(characterId, fileName) {
// Make sure all the data is loaded.
await unshallowCharacter(characterId);
/** @type {import('./scripts/char-data.js').v1CharData} */
/** @type {Character} */
const character = characters[characterId];
if (!character) {
console.warn(`Character with ID ${characterId} not found.`);
@@ -6731,7 +6730,7 @@ export async function renameCharacter(name = null, { silent = false, renameChats
}
// Also rename as a group member
await renameGroupMember(oldAvatar, newAvatar, newValue);
await renameGroupMember(oldAvatar, newAvatar, newValue.toString());
const renamePastChatsConfirm = renameChats !== null
? renameChats
: silent
@@ -6860,6 +6859,11 @@ export function saveChatDebounced() {
* @returns {Promise<void>}
*/
export async function saveChat({ chatName, withMetadata, mesId, force = false } = {}) {
if (selected_group) {
toastr.error(t`Operation was aborted to prevent data corruption.`, t`saveChat called for a group chat`);
throw new Error('saveChat called for a group chat');
}
if (arguments.length > 0 && typeof arguments[0] !== 'object') {
console.trace('saveChat called with positional arguments. Please use an object instead.');
[chatName, withMetadata, mesId, force] = arguments;
@@ -6879,26 +6883,15 @@ export async function saveChat({ chatName, withMetadata, mesId, force = false }
}
characters[this_chid]['date_last_chat'] = Date.now();
chat.forEach(function (item, i) {
if (item['is_group']) {
toastr.error(t`Trying to save group chat with regular saveChat function. Aborting to prevent corruption.`);
throw new Error('Group chat saved from saveChat');
}
});
const trimmedChat = (mesId !== undefined && mesId >= 0 && mesId < chat.length)
? chat.slice(0, Number(mesId) + 1)
: chat.slice();
const chatToSave = [
{
user_name: name1,
character_name: name2,
create_date: chat_create_date,
chat_metadata: metadata,
},
...trimmedChat,
];
/** @type {ChatHeader} */
const chatHeader = {
chat_metadata: metadata,
};
try {
const result = await fetch('/api/chats/save', {
@@ -6908,7 +6901,7 @@ export async function saveChat({ chatName, withMetadata, mesId, force = false }
body: JSON.stringify({
ch_name: characters[this_chid].name,
file_name: fileName,
chat: chatToSave,
chat: [chatHeader, ...trimmedChat],
avatar_url: characters[this_chid].avatar,
force: force,
}),
@@ -7082,7 +7075,7 @@ export async function unshallowCharacter(characterId) {
return;
}
/** @type {import('./scripts/char-data.js').v1CharData} */
/** @type {Character} */
const character = characters[characterId];
if (!character) {
console.debug('Character not found:', characterId);
@@ -7121,13 +7114,10 @@ export async function getChat() {
});
if (response[0] !== undefined) {
chat.splice(0, chat.length, ...response);
chat_create_date = chat[0]['create_date'];
chat_metadata = chat[0]['chat_metadata'] ?? {};
chat.shift();
chat.forEach(ensureMessageMediaIsArray);
} else {
chat_create_date = humanizedDateTime();
}
if (!chat_metadata['integrity']) {
chat_metadata['integrity'] = uuidv4();
@@ -8720,12 +8710,7 @@ export async function deleteSwipe(swipeId = null, messageId = chat.length - 1) {
}
export async function saveMetadata() {
if (selected_group) {
await editGroup(selected_group, true, false);
}
else {
await saveChatConditional();
}
return await saveChatConditional();
}
export async function saveChatConditional() {
@@ -10356,7 +10341,7 @@ jQuery(async function () {
});
$('#rm_button_selected_ch').on('click', function () {
if (selected_group) {
select_group_chats(selected_group);
select_group_chats(selected_group, false);
} else {
selected_button = 'character_edit';
select_selected_character(this_chid);
@@ -11289,7 +11274,7 @@ jQuery(async function () {
$('#rm_button_group_chats').on('click', function () {
selected_button = 'group_chats';
select_group_chats();
select_group_chats(null, false);
});
$('#rm_button_back_from_group').on('click', function () {
public/scripts/bookmarks.js
+29 -28
View File
@@ -11,6 +11,7 @@ import {
chat,
saveChatConditional,
saveItemizedPrompts,
setActiveGroup,
} from '../script.js';
import { humanizedDateTime } from './RossAscends-mods.js';
import {
@@ -298,28 +299,31 @@ export async function convertSoloToGroupChat() {
const chats = [chatName];
const members = [character.avatar];
const favChecked = character.fav || character.fav == 'true';
/** @type {any} */
/** @type {ChatMetadata} */
const metadata = Object.assign({}, chat_metadata);
delete metadata.main_chat;
/** @type {ChatHeader} */
const chatHeader = { chat_metadata: metadata };
/** @type {Omit<Group, 'id'>} */
const groupCreateModel = {
name: name,
members: members,
avatar_url: avatar,
allow_self_responses: false,
activation_strategy: group_activation_strategy.NATURAL,
disabled_members: [],
fav: favChecked,
chat_id: chatName,
chats: chats,
hideMutedSprites: false,
generation_mode: group_generation_mode.SWAP,
auto_mode_delay: DEFAULT_AUTO_MODE_DELAY,
};
const createGroupResponse = await fetch('/api/groups/create', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
name: name,
members: members,
avatar_url: avatar,
allow_self_responses: false,
activation_strategy: group_activation_strategy.NATURAL,
disabled_members: [],
chat_metadata: metadata,
fav: favChecked,
chat_id: chatName,
chats: chats,
hideMutedSprites: false,
generation_mode: group_generation_mode.SWAP,
auto_mode_delay: DEFAULT_AUTO_MODE_DELAY,
}),
body: JSON.stringify(groupCreateModel),
});
if (!createGroupResponse.ok) {
@@ -327,6 +331,7 @@ export async function convertSoloToGroupChat() {
return;
}
/** @type {Group} */
const group = await createGroupResponse.json();
// Convert tags list and assign to group
@@ -336,39 +341,34 @@ export async function convertSoloToGroupChat() {
await getCharacters();
// Convert chat to group format
const groupChat = chat.slice();
const groupChat = [...chat].map(m => structuredClone(m));
const genIdFirst = Date.now();
for (let index = 0; index < groupChat.length; index++) {
const message = groupChat[index];
// Save group-chat marker
if (index == 0) {
// @ts-ignore
message.is_group = true;
}
// Skip messages we don't care about
if (message.is_user || message.is_system || message.extra?.type === system_message_types.NARRATOR || message.force_avatar !== undefined) {
continue;
}
if (!message.extra || typeof message.extra !== 'object') {
message.extra = {};
}
// Set force fields for solo character
message.name = character.name;
message.original_avatar = character.avatar;
message.force_avatar = getThumbnailUrl('avatar', character.avatar);
// Allow regens of a single message in group
if (typeof message.extra !== 'object') {
message.extra = { gen_id: genIdFirst + index };
}
message.extra.gen_id = genIdFirst + index;
}
// Save group chat
const createChatResponse = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chatName, chat: groupChat }),
body: JSON.stringify({ id: chatName, chat: [chatHeader, ...groupChat] }),
});
if (!createChatResponse.ok) {
@@ -378,6 +378,7 @@ export async function convertSoloToGroupChat() {
}
// Click on the freshly selected group to open it
setActiveGroup(group.id);
await openGroupById(group.id);
toastr.success(t`The chat has been successfully converted!`);
public/scripts/extensions/attachments/index.js
+1 -1
View File
@@ -218,7 +218,7 @@ function cleanUpAttachments() {
/**
* Clean up character attachments when a character is deleted.
* @param {{character: import('../../char-data.js').v1CharData}} data Event data
* @param {{character: Character}} data Event data
*/
function cleanUpCharacterAttachments(data) {
const avatar = data?.character?.avatar;
public/scripts/extensions/gallery/index.js
+1 -1
View File
@@ -95,7 +95,7 @@ function initSettings() {
/**
* Retrieves the gallery folder for a given character.
* @param {import('../../char-data.js').v1CharData} char Character data
* @param {Character} char Character data
* @returns {string} The gallery folder for the character
*/
function getGalleryFolder(char) {
public/scripts/extensions/quick-reply/index.js
+1 -1
View File
@@ -152,7 +152,7 @@ const handleCharChange = () => {
lastCharId = this_chid;
// If no character is loaded, there's nothing more to do.
/** @type {import('../../char-data.js').v1CharData} */
/** @type {Character} */
const character = characters[this_chid];
if (!character || selected_group) {
return;
public/scripts/extensions/regex/engine.js
+3 -3
View File
@@ -105,7 +105,7 @@ export async function saveScriptsByType(scripts, scriptType) {
/**
* Check if character's regexes are allowed to be used; if character is undefined, returns false
* @param {import('../../char-data.js').v1CharData|undefined} character
* @param {Character|undefined} character
* @returns {boolean}
*/
export function isScopedScriptsAllowed(character) {
@@ -114,7 +114,7 @@ export function isScopedScriptsAllowed(character) {
/**
* Allow character's regexes to be used; if character is undefined, do nothing
* @param {import('../../char-data.js').v1CharData|undefined} character
* @param {Character|undefined} character
* @returns {void}
*/
export function allowScopedScripts(character) {
@@ -133,7 +133,7 @@ export function allowScopedScripts(character) {
/**
* Disallow character's regexes to be used; if character is undefined, do nothing
* @param {import('../../char-data.js').v1CharData|undefined} character
* @param {Character|undefined} character
* @returns {void}
*/
export function disallowScopedScripts(character) {
public/scripts/group-chats.js
+306 -86
View File
@@ -17,6 +17,8 @@ import {
renderPaginationDropdown,
paginationDropdownChangeHandler,
waitUntilCondition,
uuidv4,
humanFileSize,
} from './utils.js';
import { RA_CountCharTokens, humanizedDateTime, dragElement, favsToHotswap, getMessageTimeStamp } from './RossAscends-mods.js';
import { power_user, loadMovingUIState, sortEntitiesList } from './power-user.js';
@@ -109,7 +111,9 @@ export {
let is_group_generating = false; // Group generation flag
let is_group_automode_enabled = false;
let hideMutedSprites = false;
/** @type {Group[]} */
let groups = [];
/** @type {string|null} */
let selected_group = null;
let group_generation_id = null;
let fav_grp_checked = false;
@@ -143,6 +147,11 @@ function setAutoModeWorker() {
autoModeWorker = setInterval(groupChatAutoModeWorker, autoModeDelay * 1000);
}
/**
* Saves a group to the server.
* @param {Group} group Group object to save
* @param {boolean} reload Whether to reload characters after saving
*/
async function _save(group, reload = true) {
await fetch('/api/groups/edit', {
method: 'POST',
@@ -179,6 +188,11 @@ async function regenerateGroup() {
generateGroupWrapper(false, 'normal', { signal: abortController.signal });
}
/**
* Loads group chat messages from the server.
* @param {string} chatId Chat ID
* @returns {Promise<ChatFile>} Array of chat messages
*/
async function loadGroupChat(chatId) {
const response = await fetch('/api/chats/group/get', {
method: 'POST',
@@ -188,12 +202,20 @@ async function loadGroupChat(chatId) {
if (response.ok) {
const data = await response.json();
if (!Array.isArray(data)) {
return [];
}
return data;
}
return [];
}
/**
* Validates a group by checking if all members exist and removing duplicates.
* @param {Group} group Group to validate
* @returns {Promise<void>}
*/
async function validateGroup(group) {
if (!group) return;
@@ -225,6 +247,12 @@ async function validateGroup(group) {
}
}
/**
* Loads the chat messages for a specific group.
* @param {string} groupId - The ID of the group to load chat messages for.
* @param {boolean} reload - Whether to reload the group chat after loading.
* @returns {Promise<void>} A promise that resolves when the chat messages have been loaded.
*/
export async function getGroupChat(groupId, reload = false) {
const group = groups.find((x) => x.id === groupId);
if (!group) {
@@ -238,9 +266,19 @@ export async function getGroupChat(groupId, reload = false) {
const chat_id = group.chat_id;
const data = await loadGroupChat(chat_id);
const metadata = group.chat_metadata ?? {};
const metadata = data?.[0]?.chat_metadata ?? {};
const freshChat = !metadata.tainted && (!Array.isArray(data) || !data.length);
// Remove chat file header if present
if (Array.isArray(data) && data.length && Object.hasOwn(data[0], 'chat_metadata')) {
data.shift();
}
// Add integrity slug if missing
if (!metadata['integrity']) {
metadata['integrity'] = uuidv4();
}
await loadItemizedPrompts(getCurrentChatId());
if (group && Array.isArray(group.members) && freshChat) {
@@ -266,7 +304,6 @@ export async function getGroupChat(groupId, reload = false) {
}
await saveGroupChat(groupId, false);
} else if (Array.isArray(data) && data.length) {
data[0].is_group = true;
chat.splice(0, chat.length, ...data);
chat.forEach(ensureMessageMediaIsArray);
chatElement.find('.mes').remove();
@@ -528,6 +565,11 @@ export function getGroupCharacterCards(groupId, characterId) {
return { description, personality, scenario, mesExamples };
}
/**
* Gets the first message for a character.
* @param {Character} character Character object
* @returns {Promise<ChatMessage>} First message object
*/
async function getFirstCharacterMessage(character) {
let messageText = character.first_mes;
@@ -566,20 +608,53 @@ function resetSelectedGroup() {
is_group_generating = false;
}
async function saveGroupChat(groupId, shouldSaveGroup) {
/**
* Saves a group chat to the server.
* @param {string} groupId Group ID
* @param {boolean} shouldSaveGroup Whether to save the group after saving the chat
* @param {boolean} force Force the saving on integrity error
* @returns {Promise<void>} A promise that resolves when the group chat has been saved.
*/
async function saveGroupChat(groupId, shouldSaveGroup, force = false) {
const group = groups.find(x => x.id == groupId);
const chat_id = group.chat_id;
group['date_last_chat'] = Date.now();
/** @type {ChatHeader} */
const chatHeader = {
chat_metadata: { ...chat_metadata },
};
const response = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chat_id, chat: [...chat] }),
body: JSON.stringify({ id: chat_id, chat: [chatHeader, ...chat], force: force }),
});
if (!response.ok) {
toastr.error(t`Check the server connection and reload the page to prevent data loss.`, t`Group Chat could not be saved`);
console.error('Group chat could not be saved', response);
return;
const errorData = await response.json();
const isIntegrityError = errorData?.error === 'integrity' && !force;
if (!isIntegrityError) {
toastr.error(t`Check the server connection and reload the page to prevent data loss.`, t`Group Chat could not be saved`);
console.error('Group chat could not be saved', response);
return;
}
const popupResult = await Popup.show.input(
t`ERROR: Chat integrity check failed while saving the file.`,
t`<p>After you click OK, the page will be reloaded to prevent data corruption.</p>
<p>To confirm an overwrite (and potentially <b>LOSE YOUR DATA</b>), enter <code>OVERWRITE</code> (in all caps) in the box below before clicking OK.</p>`,
'',
{ okButton: 'OK', cancelButton: false },
);
const forceSaveConfirmed = popupResult === 'OVERWRITE';
if (!forceSaveConfirmed) {
console.warn('Chat integrity check failed, and user did not confirm the overwrite. Reloading the page.');
window.location.reload();
return;
}
await saveGroupChat(groupId, shouldSaveGroup, true);
}
if (shouldSaveGroup) {
@@ -587,6 +662,12 @@ async function saveGroupChat(groupId, shouldSaveGroup) {
}
}
/**
* Renames a group member across all groups and their chats.
* @param {string} oldAvatar Old avatar name
* @param {string} newAvatar New avatar name
* @param {string} newName New character name
*/
export async function renameGroupMember(oldAvatar, newAvatar, newName) {
// Scan every group for our renamed character
for (const group of groups) {
@@ -614,6 +695,11 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
if (Array.isArray(messages) && messages.length) {
// Iterate over every chat message
for (const message of messages) {
// Skip the chat header
if (Object.hasOwn(message, 'chat_metadata')) {
continue;
}
// Only look at character messages
if (message.is_user || message.is_system) {
continue;
@@ -652,6 +738,9 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
}
}
/**
* Fetches all groups from the server and processes them.
*/
async function getGroups() {
const response = await fetch('/api/groups/all', {
method: 'POST',
@@ -659,8 +748,9 @@ async function getGroups() {
});
if (response.ok) {
/** @type {Group[]} */
const data = await response.json();
groups = data.sort((a, b) => a.id - b.id);
groups = data.slice();
// Convert groups to new format
for (const group of groups) {
@@ -678,9 +768,6 @@ async function getGroups() {
.filter(x => x)
.filter(onlyUnique);
}
if (group.past_metadata == undefined) {
group.past_metadata = {};
}
if (typeof group.chat_id === 'number') {
group.chat_id = String(group.chat_id);
}
@@ -691,6 +778,11 @@ async function getGroups() {
}
}
/**
* Gets a group UI block for the list.
* @param {Group} group Group object
* @returns {JQuery<HTMLElement>} jQuery element representing the group block
*/
export function getGroupBlock(group) {
let count = 0;
let namesList = [];
@@ -712,7 +804,7 @@ export function getGroupBlock(group) {
template.find('.ch_name').text(group.name).attr('title', `[Group] ${group.name}`);
template.find('.group_fav_icon').css('display', 'none');
template.addClass(group.fav ? 'is_fav' : '');
template.find('.ch_fav').val(group.fav);
template.find('.ch_fav').val(String(group.fav));
template.find('.group_select_counter').text(count + ' ' + (count != 1 ? t`characters` : t`character`));
template.find('.group_select_block_list').text(namesList.join(', '));
@@ -728,6 +820,10 @@ export function getGroupBlock(group) {
return template;
}
/**
* Updates the avatar display for a given group.
* @param {Group} group Group object
*/
function updateGroupAvatar(group) {
$('#group_avatar_preview').empty().append(getGroupAvatar(group));
@@ -740,7 +836,11 @@ function updateGroupAvatar(group) {
favsToHotswap();
}
// check if isDataURLor if it's a valid local file url
/**
* Checks if a URL is a valid image URL.
* @param {string} url URL to check
* @returns {boolean} True if valid, false otherwise
*/
function isValidImageUrl(url) {
// check if empty dict
if (Object.keys(url).length === 0) {
@@ -749,6 +849,11 @@ function isValidImageUrl(url) {
return isDataURL(url) || (url && (url.startsWith('user') || url.startsWith('/user')));
}
/**
* Gets a group avatar element.
* @param {Group} group Group object
* @returns {JQuery<HTMLElement>} Group avatar element
*/
function getGroupAvatar(group) {
if (!group) {
return $(`<div class="avatar"><img src="${default_avatar}"></div>`);
@@ -797,6 +902,11 @@ function getGroupAvatar(group) {
return groupAvatar;
}
/**
* Gets chat IDs for a group.
* @param {string} groupId Group ID
* @returns {string[]} Array of chat IDs
*/
function getGroupChatNames(groupId) {
const group = groups.find(x => x.id === groupId);
@@ -811,7 +921,14 @@ function getGroupChatNames(groupId) {
return names;
}
async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
/**
* Generates text for the group chat by queueing members according to the activation strategy.
* @param {boolean} byAutoMode If the generation was triggered by the auto mode.
* @param {string?} type Generation type
* @param {object} params Additional Generate parameters
* @returns {Promise<string|void>} Generated text or nothing if no generation occurred
*/
async function generateGroupWrapper(byAutoMode, type = null, params = {}) {
function throwIfAborted() {
if (params.signal instanceof AbortSignal && params.signal.aborted) {
throw new Error('AbortSignal was fired. Group generation stopped');
@@ -830,7 +947,7 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
// Auto-navigate back to group menu
if (menu_type !== 'group_edit') {
select_group_chats(selected_group);
select_group_chats(selected_group, false);
await delay(1);
}
@@ -859,7 +976,7 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
let activationText = '';
let isUserInput = false;
if (userInput?.length && !by_auto_mode) {
if (userInput?.length && !byAutoMode) {
isUserInput = true;
activationText = userInput;
} else {
@@ -935,12 +1052,12 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
// Wait for generation to finish
const generateType = ['swipe', 'impersonate', 'quiet', 'continue'].includes(type) ? type : 'normal';
textResult = await Generate(generateType, { automatic_trigger: by_auto_mode, ...(params || {}) });
textResult = await Generate(generateType, { automatic_trigger: byAutoMode, ...(params || {}) });
let messageChunk = textResult?.messageChunk;
if (messageChunk) {
while (shouldAutoContinue(messageChunk, type === 'impersonate')) {
textResult = await Generate('continue', { automatic_trigger: by_auto_mode, ...(params || {}) });
textResult = await Generate('continue', { automatic_trigger: byAutoMode, ...(params || {}) });
messageChunk = textResult?.messageChunk;
}
}
@@ -966,6 +1083,10 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
return Promise.resolve(textResult);
}
/**
* Gets the generation ID of the last chat message.
* @returns {number|null} Generation ID or null
*/
function getLastMessageGenerationId() {
let generationId = null;
if (chat.length > 0) {
@@ -977,6 +1098,11 @@ function getLastMessageGenerationId() {
return generationId;
}
/**
* Activate group chat members for 'impersonate' generation type.
* @param {string[]} members Array of group member avatar ids
* @returns {number[]} Array of character ids
*/
function activateImpersonate(members) {
const randomIndex = Math.floor(Math.random() * members.length);
const activatedMembers = [members[randomIndex]];
@@ -1039,6 +1165,11 @@ function activateSwipe(members, { allowSystem = false } = {}) {
return memberIds;
}
/**
* Activate group members for the list activation order.
* @param {string[]} members Array of group member avatar ids
* @returns {number[]} Array of character ids
*/
function activateListOrder(members) {
let activatedMembers = members.filter(onlyUnique);
@@ -1092,6 +1223,15 @@ function activatePooledOrder(members, lastMessage, isUserInput) {
return memberId !== -1 ? [memberId] : [];
}
/**
* Activate group members for the natural activation order.
* @param {string[]} members Array of group member avatar ids
* @param {string} input User input that triggered the generation
* @param {ChatMessage} lastMessage Last message in the chat
* @param {boolean} allowSelfResponses If the group allows self-responses
* @param {boolean} isUserInput If the generation was triggered by user input
* @returns {number[]} Array of character ids
*/
function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, isUserInput) {
let activatedMembers = [];
@@ -1168,6 +1308,11 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i
return memberIds;
}
/**
* Deletes a group from the server by ID.
* @param {string} id Group ID to delete
* @returns {Promise<void>} Promise that resolves when the group is deleted
*/
async function deleteGroup(id) {
const group = groups.find((x) => x.id === id);
@@ -1197,6 +1342,13 @@ async function deleteGroup(id) {
}
}
/**
* Edits a group by ID.
* @param {string} id Group ID to edit
* @param {boolean} immediately Whether to save immediately
* @param {boolean} reload Whether to reload the groups after saving
* @returns {Promise<void>} Promise that resolves when the group is edited
*/
export async function editGroup(id, immediately, reload = true) {
let group = groups.find((x) => x.id === id);
@@ -1204,11 +1356,6 @@ export async function editGroup(id, immediately, reload = true) {
return;
}
if (id === selected_group) {
// structuredClone may cause issues if metadata has non-cloneable references
group['chat_metadata'] = JSON.parse(JSON.stringify(chat_metadata));
}
if (immediately) {
return await _save(group, reload);
}
@@ -1260,6 +1407,12 @@ async function groupChatAutoModeWorker() {
await generateGroupWrapper(true, 'auto', { signal: groupAutoModeAbortController.signal });
}
/**
* Modifies a group member by adding or removing them.
* @param {string} groupId Group ID
* @param {JQuery<HTMLElement>} groupMember Group member element
* @param {boolean} isDelete If true, removes the member; otherwise adds the member
*/
async function modifyGroupMember(groupId, groupMember, isDelete) {
const id = groupMember.data('id');
const thisGroup = groups.find((x) => x.id == groupId);
@@ -1287,9 +1440,16 @@ async function modifyGroupMember(groupId, groupMember, isDelete) {
$('#rm_group_submit').prop('disabled', !groupHasMembers);
}
async function reorderGroupMember(chat_id, groupMember, direction) {
/**
* Reorders a group member up or down.
* @param {string} groupId Group ID
* @param {JQuery<HTMLElement>} groupMember Group member element
* @param {string} direction Direction to move the member ('up' or 'down')
* @returns {Promise<void>} Promise that resolves when the member has been reordered
*/
async function reorderGroupMember(groupId, groupMember, direction) {
const id = groupMember.data('id');
const thisGroup = groups.find((x) => x.id == chat_id);
const thisGroup = groups.find((x) => x.id == groupId);
const memberArray = thisGroup?.members ?? newGroupMembers;
const indexOf = memberArray.indexOf(id);
@@ -1312,7 +1472,7 @@ async function reorderGroupMember(chat_id, groupMember, direction) {
// Existing groups need to modify members list
if (openGroupId) {
await editGroup(chat_id, false, false);
await editGroup(groupId, false, false);
updateGroupAvatar(thisGroup);
}
}
@@ -1358,10 +1518,16 @@ async function onGroupNameInput() {
let _thisGroup = groups.find((x) => x.id == openGroupId);
_thisGroup.name = $(this).val();
$('#rm_button_selected_ch').children('h2').text(_thisGroup.name);
await editGroup(openGroupId);
await editGroup(openGroupId, false);
}
}
/**
* Checks if a character with the given avatar ID is a member of the group.
* @param {Group} group Group object
* @param {string} avatarId Avatar ID to check
* @returns {boolean} True if the avatar is a member of the group, false otherwise
*/
function isGroupMember(group, avatarId) {
if (group && Array.isArray(group.members)) {
return group.members.includes(avatarId);
@@ -1370,6 +1536,13 @@ function isGroupMember(group, avatarId) {
}
}
/**
* Gets group characters based on filters.
* @param {object} param
* @param {boolean} [param.doFilter=false] Whether to apply filters
* @param {boolean} [param.onlyMembers=false] Whether to include only group members
* @returns {Array<{item: Character, id: number, type: string}>} Array of group character objects
*/
function getGroupCharacters({ doFilter = false, onlyMembers = false } = {}) {
function sortMembersFn(a, b) {
const membersArray = thisGroup?.members ?? newGroupMembers;
@@ -1461,15 +1634,20 @@ function printGroupMembers() {
});
}
/**
* Creates a jQuery element representing a group character block.
* @param {Character} character Character object
* @returns {JQuery<HTMLElement>} jQuery element representing the group character block
*/
function getGroupCharacterBlock(character) {
const avatar = getThumbnailUrl('avatar', character.avatar);
const template = $('#group_member_template .group_member').clone();
const isFav = character.fav || character.fav == 'true';
const isFav = !!character.fav || character.fav == 'true';
template.data('id', character.avatar);
template.find('.avatar img').attr({ 'src': avatar, 'title': character.avatar });
template.find('.ch_name').text(character.name);
template.attr('data-chid', characters.indexOf(character));
template.find('.ch_fav').val(isFav);
template.find('.ch_fav').val(String(isFav));
template.toggleClass('is_fav', isFav);
const auxFieldName = power_user.aux_field || 'character_version';
@@ -1503,6 +1681,11 @@ function getGroupCharacterBlock(character) {
return template;
}
/**
* Checks if a group member is disabled.
* @param {string} avatarId Avatar ID of the group member
* @returns {boolean} True if the group member is disabled, false otherwise
*/
function isGroupMemberDisabled(avatarId) {
const thisGroup = openGroupId && groups.find((x) => x.id == openGroupId);
return Boolean(thisGroup && thisGroup.disabled_members.includes(avatarId));
@@ -1553,6 +1736,11 @@ async function onHideMutedSpritesClick(value) {
}
}
/**
* Toggles the visibility of hidden controls based on the group's generation mode.
* @param {Group} group Group object
* @param {number|null} generationMode Generation mode, or null to use the group's current generation mode
*/
function toggleHiddenControls(group, generationMode = null) {
const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);
$('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);
@@ -1564,6 +1752,11 @@ function toggleHiddenControls(group, generationMode = null) {
}
}
/**
* Opens a group creation/editing right menu.
* @param {string|null} groupId ID of the group to select or null if creating a new group
* @param {boolean} skipAnimation If true, skips the animation when selecting the group
*/
function select_group_chats(groupId, skipAnimation) {
openGroupId = groupId;
newGroupMembers = [];
@@ -1774,6 +1967,11 @@ function updateFavButtonState(state) {
$('#group_favorite_button').toggleClass('fav_off', !fav_grp_checked);
}
/**
* Opens a group chat by its ID and updates the UI accordingly.
* @param {string} groupId ID of the group to open
* @returns {Promise<boolean>} Whether the group was opened
*/
export async function openGroupById(groupId) {
if (isChatSaving) {
toastr.info(t`Please wait until the chat is saved before switching characters.`, t`Your chat is still saving...`);
@@ -1786,7 +1984,7 @@ export async function openGroupById(groupId) {
}
if (!is_send_press && !is_group_generating) {
select_group_chats(groupId);
select_group_chats(groupId, false);
if (selected_group !== groupId) {
groupChatQueueOrder = new Map();
@@ -1806,6 +2004,11 @@ export async function openGroupById(groupId) {
return false;
}
/**
* Peeks the character definition from a group member element.
* @param {JQuery<HTMLElement>} characterSelect Character select element
* @returns {Promise<void>}
*/
async function openCharacterDefinition(characterSelect) {
if (is_group_generating) {
toastr.warning(t`Can't peek a character while group reply is being generated`);
@@ -1834,7 +2037,7 @@ function filterGroupMembers() {
}
async function createGroup() {
let name = $('#rm_group_chat_name').val();
let name = $('#rm_group_chat_name').val().toString();
let allowSelfResponses = !!$('#rm_group_allow_self_responses').prop('checked');
let activationStrategy = Number($('#rm_group_activation_strategy').find(':selected').val()) ?? group_activation_strategy.NATURAL;
let generationMode = Number($('#rm_group_generation_mode').find(':selected').val()) ?? group_generation_mode.SWAP;
@@ -1846,29 +2049,30 @@ async function createGroup() {
name = t`Group: ${memberNames}`;
}
const avatar_url = $('#group_avatar_preview img').attr('src');
const avatarUrl = $('#group_avatar_preview img').attr('src');
const chatName = humanizedDateTime();
const chats = [chatName];
/** @type {Omit<Group, 'id'>} */
const groupCreateModel = {
name: name,
members: members,
avatar_url: isValidImageUrl(avatarUrl) ? avatarUrl : default_avatar,
allow_self_responses: allowSelfResponses,
hideMutedSprites: hideMutedSprites,
activation_strategy: activationStrategy,
generation_mode: generationMode,
disabled_members: [],
fav: fav_grp_checked,
chat_id: chatName,
chats: chats,
auto_mode_delay: autoModeDelay,
};
const createGroupResponse = await fetch('/api/groups/create', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
name: name,
members: members,
avatar_url: isValidImageUrl(avatar_url) ? avatar_url : default_avatar,
allow_self_responses: allowSelfResponses,
hideMutedSprites: hideMutedSprites,
activation_strategy: activationStrategy,
generation_mode: generationMode,
disabled_members: [],
chat_metadata: {},
fav: fav_grp_checked,
chat_id: chatName,
chats: chats,
auto_mode_delay: autoModeDelay,
}),
body: JSON.stringify(groupCreateModel),
});
if (createGroupResponse.ok) {
@@ -1880,6 +2084,11 @@ async function createGroup() {
}
}
/**
* Creates a new group chat within the specified group.
* @param {string} groupId Group ID
* @returns {Promise<void>} Promise that resolves when the new group chat is created
*/
export async function createNewGroupChat(groupId) {
const group = groups.find(x => x.id === groupId);
@@ -1887,27 +2096,22 @@ export async function createNewGroupChat(groupId) {
return;
}
const oldChatName = group.chat_id;
const newChatName = humanizedDateTime();
if (typeof group.past_metadata !== 'object') {
group.past_metadata = {};
}
await clearChat();
chat.length = 0;
if (oldChatName) {
group.past_metadata[oldChatName] = Object.assign({}, chat_metadata);
}
const newChatName = humanizedDateTime();
group.chats.push(newChatName);
group.chat_id = newChatName;
group.chat_metadata = {};
updateChatMetadata(group.chat_metadata, true);
updateChatMetadata({}, true);
await editGroup(group.id, true, false);
await getGroupChat(group.id);
}
/**
* Retrieves past chats for a specified group.
* @param {string} groupId Group ID
* @returns {Promise<Array>} Array of past chats
*/
export async function getGroupPastChats(groupId) {
const group = groups.find(x => x.id === groupId);
@@ -1920,16 +2124,22 @@ export async function getGroupPastChats(groupId) {
try {
for (const chatId of group.chats) {
const messages = await loadGroupChat(chatId);
let this_chat_file_size = (JSON.stringify(messages).length / 1024).toFixed(2) + 'kb';
let chat_items = messages.length;
if (!Array.isArray(messages)) {
continue;
}
const fileSize = humanFileSize(JSON.stringify(messages).length);
if (messages.length > 0 && Object.hasOwn(messages[0], 'chat_metadata')) {
messages.shift();
}
const chatItems = messages.length;
const lastMessage = messages.length ? messages[messages.length - 1].mes : '[The chat is empty]';
const lastMessageDate = messages.length ? (messages[messages.length - 1].send_date || Date.now()) : Date.now();
chats.push({
'file_name': chatId,
'mes': lastMessage,
'last_mes': lastMessageDate,
'file_size': this_chat_file_size,
'chat_items': chat_items,
'file_size': fileSize,
'chat_items': chatItems,
});
}
} catch (err) {
@@ -1938,6 +2148,12 @@ export async function getGroupPastChats(groupId) {
return chats;
}
/**
* Opens a specific group chat for the specified group by its ID.
* @param {string} groupId Group ID
* @param {string} chatId Chat ID
* @returns {Promise<void>}
*/
export async function openGroupChat(groupId, chatId) {
await waitUntilCondition(() => !isChatSaving, debounce_timeout.extended, 10);
const group = groups.find(x => x.id === groupId);
@@ -1948,17 +2164,21 @@ export async function openGroupChat(groupId, chatId) {
await clearChat();
chat.length = 0;
const previousChat = group.chat_id;
group.past_metadata[previousChat] = Object.assign({}, chat_metadata);
group.chat_id = chatId;
group.chat_metadata = group.past_metadata[chatId] || {};
group['date_last_chat'] = Date.now();
updateChatMetadata(group.chat_metadata, true);
updateChatMetadata({}, true);
await editGroup(groupId, true, false);
await getGroupChat(groupId);
}
/**
* Renames a group chat within the specified group.
* @param {string} groupId Group ID
* @param {string} oldChatId Old chat ID
* @param {string} newChatId New chat ID
* @returns {Promise<void>} Promise that resolves when the group chat is renamed
*/
export async function renameGroupChat(groupId, oldChatId, newChatId) {
const group = groups.find(x => x.id === groupId);
@@ -1972,8 +2192,6 @@ export async function renameGroupChat(groupId, oldChatId, newChatId) {
group.chats.splice(group.chats.indexOf(oldChatId), 1);
group.chats.push(newChatId);
group.past_metadata[newChatId] = (group.past_metadata[oldChatId] || {});
delete group.past_metadata[oldChatId];
await editGroup(groupId, true, true);
}
@@ -1990,12 +2208,7 @@ export async function deleteGroupChatByName(groupId, chatName) {
return;
}
if (typeof group.past_metadata !== 'object') {
group.past_metadata = {};
}
group.chats.splice(group.chats.indexOf(chatName), 1);
delete group.past_metadata[chatName];
const response = await fetch('/api/chats/group/delete', {
method: 'POST',
@@ -2011,12 +2224,8 @@ export async function deleteGroupChatByName(groupId, chatName) {
// If the deleted chat was the current chat, switch to the last chat in the group
if (group.chat_id === chatName) {
group.chat_id = '';
group.chat_metadata = {};
const newChatName = group.chats.length ? group.chats[group.chats.length - 1] : humanizedDateTime();
group.chat_id = newChatName;
group.chat_metadata = group.past_metadata[newChatName] || {};
}
await editGroup(groupId, true, true);
@@ -2038,12 +2247,10 @@ export async function deleteGroupChat(groupId, chatId, { jumpToNewChat = true }
}
group.chats.splice(group.chats.indexOf(chatId), 1);
delete group.past_metadata[chatId];
if (group.chat_id === chatId) {
group.chat_id = '';
group.chat_metadata = {};
updateChatMetadata(group.chat_metadata, true);
updateChatMetadata({}, true);
}
const response = await fetch('/api/chats/group/delete', {
@@ -2101,6 +2308,14 @@ export async function importGroupChat(formData, { refresh = true } = {}) {
return [];
}
/**
* Saves the current group chat as a bookmark chat.
* @param {string} groupId Group ID
* @param {string} name Name of the chat to save
* @param {ChatMetadata?} metadata New metadata to save with the chat
* @param {number|undefined} mesId Optional message ID to trim the chat up to
* @returns {Promise<void>} Promise that resolves when the group chat is saved
*/
export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) {
const group = groups.find(x => x.id === groupId);
@@ -2108,11 +2323,16 @@ export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) {
return;
}
group.past_metadata[name] = { ...chat_metadata, ...(metadata || {}) };
group.chats.push(name);
const trimmed_chat = (mesId !== undefined && mesId >= 0 && mesId < chat.length)
? chat.slice(0, parseInt(mesId) + 1)
/** @type {ChatHeader} */
const chatHeader = {
chat_metadata: { ...chat_metadata, ...(metadata || {}) },
};
/** @type {ChatMessage[]} */
const trimmedChat = (mesId !== undefined && mesId >= 0 && mesId < chat.length)
? chat.slice(0, Number(mesId) + 1)
: chat;
await editGroup(groupId, true, false);
@@ -2120,7 +2340,7 @@ export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) {
const response = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: name, chat: [...trimmed_chat] }),
body: JSON.stringify({ id: name, chat: [chatHeader, ...trimmedChat] }),
});
if (!response.ok) {
public/scripts/utils.js
+1 -1
View File
@@ -2558,7 +2558,7 @@ export function findPersona({ name = null, allowAvatar = true, insensitive = tru
* @param {string[]?} [options.filteredByTags=null] - Tags to filter characters by
* @param {boolean} [options.preferCurrentChar=true] - Whether to prefer the current character(s)
* @param {boolean} [options.quiet=false] - Whether to suppress warnings
* @returns {import('./char-data.js').v1CharData?} - The found character or null if not found
* @returns {Character?} - The found character or null if not found
*/
export function findChar({ name = null, allowAvatar = true, insensitive = true, filteredByTags = null, preferCurrentChar = true, quiet = false } = {}) {
const matches = (char) => !name || (allowAvatar && char.avatar === name) || (insensitive ? equalsIgnoreCaseAndAccents(char.name, name) : char.name === name);
public/scripts/welcome-screen.js
+2 -2
View File
@@ -106,7 +106,7 @@ async function unshallowPermanentAssistant() {
/**
* Returns a greeting message for the assistant based on the character.
* @param {import('./char-data.js').v1CharData} character Character data
* @param {Character} character Character data
* @returns {string} Greeting message
*/
function getAssistantGreeting(character) {
@@ -623,7 +623,7 @@ export function assignCharacterAsAssistant(characterId) {
if (characterId === undefined) {
return;
}
/** @type {import('./char-data.js').v1CharData} */
/** @type {Character} */
const character = characters[characterId];
if (!character) {
return;
src/endpoints/characters.js
+1 -1
View File
@@ -1383,7 +1383,7 @@ router.post('/chats', validateAvatarUrlMiddleware, async function (request, resp
const jsonFilesPromise = jsonFiles.map((file) => {
const withMetadata = !!request.body.metadata;
const pathToFile = path.join(request.user.directories.chats, characterDirectory, file);
return getChatInfo(pathToFile, {}, false, withMetadata);
return getChatInfo(pathToFile, {}, withMetadata);
});
const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
src/endpoints/chats.js
+37 -29
View File
@@ -367,11 +367,10 @@ async function checkChatIntegrity(filePath, integritySlug) {
* Reads the information from a chat file.
* @param {string} pathToFile - Path to the chat file
* @param {object} additionalData - Additional data to include in the result
* @param {boolean} isGroup - Whether the chat is a group chat
* @param {boolean} withMetadata - Whether to read chat metadata
* @returns {Promise<ChatInfo>}
*/
export async function getChatInfo(pathToFile, additionalData = {}, isGroup = false, withMetadata = false) {
export async function getChatInfo(pathToFile, additionalData = {}, withMetadata = false) {
return new Promise(async (res) => {
const parsedPath = path.parse(pathToFile);
const stats = await fs.promises.stat(pathToFile);
@@ -387,13 +386,7 @@ export async function getChatInfo(pathToFile, additionalData = {}, isGroup = fal
...additionalData,
};
if (stats.size === 0 && !isGroup) {
console.warn(`Found an empty chat file: ${pathToFile}`);
res({});
return;
}
if (stats.size === 0 && isGroup) {
if (stats.size === 0) {
res(chatData);
return;
}
@@ -422,7 +415,7 @@ export async function getChatInfo(pathToFile, additionalData = {}, isGroup = fal
if (lastLine) {
const jsonData = tryParse(lastLine);
if (jsonData && (jsonData.name || jsonData.character_name || jsonData.chat_metadata)) {
chatData.chat_items = isGroup ? itemCounter : (itemCounter - 1);
chatData.chat_items = (itemCounter - 1);
chatData.mes = jsonData['mes'] || '[The message is empty]';
chatData.last_mes = jsonData['send_date'] || stats.mtimeMs;
@@ -774,23 +767,38 @@ router.post('/group/delete', (request, response) => {
return response.send({ error: true });
});
router.post('/group/save', (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
router.post('/group/save', async (request, response) => {
try{
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const filePath = path.join(request.user.directories.groupChats, sanitize(`${id}.jsonl`));
if (!fs.existsSync(request.user.directories.groupChats)) {
fs.mkdirSync(request.user.directories.groupChats, { recursive: true });
}
const chatData = request.body.chat;
const jsonlData = chatData.map(JSON.stringify).join('\n');
if (checkIntegrity && !request.body.force) {
const integritySlug = chatData?.[0]?.chat_metadata?.integrity;
const isIntact = await checkChatIntegrity(filePath, integritySlug);
if (!isIntact) {
console.error(`Chat integrity check failed for ${filePath}`);
return response.status(400).send({ error: 'integrity' });
}
}
writeFileAtomicSync(filePath, jsonlData, 'utf8');
getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
return response.send({ ok: true });
} catch (error) {
console.error(error);
return response.send({ error: true });
}
const id = request.body.id;
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
if (!fs.existsSync(request.user.directories.groupChats)) {
fs.mkdirSync(request.user.directories.groupChats);
}
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
return response.send({ ok: true });
});
router.post('/search', validateAvatarUrlMiddleware, function (request, response) {
@@ -983,10 +991,10 @@ router.post('/recent', async function (request, response) {
const max = parseInt(request.body.max ?? Number.MAX_SAFE_INTEGER);
const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, max);
const jsonFilesPromise = recentChats.map((file) => {
const withMetadata = Boolean(request.body.metadata);
const withMetadata = !!request.body.metadata;
return file.groupId
? getChatInfo(file.filePath, { group: file.groupId }, true, withMetadata)
: getChatInfo(file.filePath, { avatar: file.pngFile }, false, withMetadata);
? getChatInfo(file.filePath, { group: file.groupId }, withMetadata)
: getChatInfo(file.filePath, { avatar: file.pngFile }, withMetadata);
});
const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
src/endpoints/data-maid.js
+13
View File
@@ -579,9 +579,11 @@ export class DataMaidService {
const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
const groupData = tryParse(fileContent);
if (groupData?.chat_metadata && filterFn(groupData.chat_metadata)) {
console.warn('Found group chat metadata in group definition - this is deprecated behavior.');
allMetadata.push(groupData.chat_metadata);
}
if (groupData?.past_metadata) {
console.warn('Found group past chat metadata in group definition - this is deprecated behavior.');
allMetadata.push(...Object.values(groupData.past_metadata).filter(filterFn));
}
} catch (error) {
@@ -590,6 +592,17 @@ export class DataMaidService {
}
}
const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
for (const file of groupChats) {
if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
const chatMetadata = chatMessages?.[0]?.chat_metadata;
if (chatMetadata && filterFn(chatMetadata)) {
allMetadata.push(chatMetadata);
}
}
}
const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
for (const directory of chatDirectories) {
if (directory.isDirectory()) {
src/endpoints/groups.js
+104 -3
View File
@@ -1,15 +1,115 @@
import fs from 'node:fs';
import { promises as fsPromises } from 'node:fs';
import path from 'node:path';
import express from 'express';
import sanitize from 'sanitize-filename';
import { sync as writeFileAtomicSync } from 'write-file-atomic';
import { sync as writeFileAtomicSync, default as writeFileAtomic } from 'write-file-atomic';
import { humanizedISO8601DateTime } from '../util.js';
import { color, humanizedISO8601DateTime, tryParse } from '../util.js';
import { getFileNameValidationFunction } from '../middleware/validateFileName.js';
export const router = express.Router();
/**
* Warns if group data contains deprecated metadata keys and removes them.
* @param {object} groupData Group data object
*/
function warnOnGroupMetadata(groupData) {
if (typeof groupData !== 'object' || groupData === null) {
return;
}
['chat_metadata', 'past_metadata'].forEach(key => {
if (Object.hasOwn(groupData, key)) {
console.warn(color.yellow(`Group JSON data for "${groupData.id}" contains deprecated key "${key}".`));
delete groupData[key];
}
});
}
/**
* Migrates group metadata to include chat metadata for each group chat instead of the group itself.
* @param {import('../users.js').UserDirectoryList[]} userDirectories Listing of all users' directories
*/
export async function migrateGroupChatsMetadataFormat(userDirectories) {
for (const userDirs of userDirectories) {
try {
let anyDataMigrated = false;
const backupPath = path.join(userDirs.backups, '_group_metadata_update');
const groupFiles = await fsPromises.readdir(userDirs.groups, { withFileTypes: true });
const groupChatFiles = await fsPromises.readdir(userDirs.groupChats, { withFileTypes: true });
for (const groupFile of groupFiles) {
try {
const isJsonFile = groupFile.isFile() && path.extname(groupFile.name) === '.json';
if (!isJsonFile) {
continue;
}
const groupFilePath = path.join(userDirs.groups, groupFile.name);
const groupDataRaw = await fsPromises.readFile(groupFilePath, 'utf8');
const groupData = tryParse(groupDataRaw) || {};
const needsMigration = ['chat_metadata', 'past_metadata'].some(key => Object.hasOwn(groupData, key));
if (!needsMigration) {
continue;
}
if (!fs.existsSync(backupPath)){
await fsPromises.mkdir(backupPath, { recursive: true });
}
await fsPromises.copyFile(groupFilePath, path.join(backupPath, groupFile.name));
const allMetadata = {
...(groupData.past_metadata || {}),
[groupData.chat_id]: (groupData.chat_metadata || {}),
};
if (!Array.isArray(groupData.chats)) {
console.warn(color.yellow(`Group ${groupFile.name} has no chats array, skipping migration.`));
continue;
}
for (const chatId of groupData.chats) {
try {
const chatFileName = sanitize(`${chatId}.jsonl`);
const chatFileDirent = groupChatFiles.find(f => f.isFile() && f.name === chatFileName);
if (!chatFileDirent) {
console.warn(color.yellow(`Group chat file ${chatId} not found, skipping migration.`));
continue;
}
const chatFilePath = path.join(userDirs.groupChats, chatFileName);
const chatMetadata = allMetadata[chatId] || {};
const chatDataRaw = await fsPromises.readFile(chatFilePath, 'utf8');
const chatData = chatDataRaw.split('\n').filter(line => line.trim()).map(line => tryParse(line)).filter(Boolean);
const alreadyHasMetadata = chatData.length > 0 && Object.hasOwn(chatData[0], 'chat_metadata');
if (alreadyHasMetadata) {
console.log(color.yellow(`Group chat ${chatId} already has chat metadata, skipping update.`));
continue;
}
await fsPromises.copyFile(chatFilePath, path.join(backupPath, chatFileName));
const chatHeader = { chat_metadata: chatMetadata };
const newChatData = [chatHeader, ...chatData];
const newChatDataRaw = newChatData.map(entry => JSON.stringify(entry)).join('\n');
await writeFileAtomic(chatFilePath, newChatDataRaw, 'utf8');
console.log(`Updated group chat data format for ${chatId}`);
anyDataMigrated = true;
} catch (chatError) {
console.error(color.red(`Could not update existing chat data for ${chatId}`), chatError);
}
}
delete groupData.chat_metadata;
delete groupData.past_metadata;
await writeFileAtomic(groupFilePath, JSON.stringify(groupData, null, 4), 'utf8');
console.log(`Migrated group chats metadata for group: ${groupData.id}`);
anyDataMigrated = true;
} catch (groupError) {
console.error(color.red(`Could not process group file ${groupFile.name}`), groupError);
}
}
if (anyDataMigrated) {
console.log(color.green(`Completed migration of group chats metadata for user at ${userDirs.root}`));
console.log(color.cyan(`Backups of modified files are located at ${backupPath}`));
}
} catch (directoryError) {
console.error(color.red(`Error migrating group chats metadata for user at ${userDirs.root}`), directoryError);
}
}
}
router.post('/all', (request, response) => {
const groups = [];
@@ -59,6 +159,7 @@ router.post('/create', (request, response) => {
return response.sendStatus(400);
}
warnOnGroupMetadata(request.body);
const id = String(Date.now());
const groupMetadata = {
id: id,
@@ -69,7 +170,6 @@ router.post('/create', (request, response) => {
activation_strategy: request.body.activation_strategy ?? 0,
generation_mode: request.body.generation_mode ?? 0,
disabled_members: request.body.disabled_members ?? [],
chat_metadata: request.body.chat_metadata ?? {},
fav: request.body.fav,
chat_id: request.body.chat_id ?? id,
chats: request.body.chats ?? [id],
@@ -92,6 +192,7 @@ router.post('/edit', getFileNameValidationFunction('id'), (request, response) =>
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
warnOnGroupMetadata(request.body);
const id = request.body.id;
const pathToFile = path.join(request.user.directories.groups, sanitize(`${id}.json`));
const fileData = JSON.stringify(request.body, null, 4);
src/server-main.js
+2
View File
@@ -68,6 +68,7 @@ import { init as settingsInit } from './endpoints/settings.js';
import { redirectDeprecatedEndpoints, ServerStartup, setupPrivateEndpoints } from './server-startup.js';
import { diskCache } from './endpoints/characters.js';
import { migrateFlatSecrets } from './endpoints/secrets.js';
import { migrateGroupChatsMetadataFormat } from './endpoints/groups.js';
// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
@@ -265,6 +266,7 @@ async function preSetupTasks() {
console.log();
const directories = await getUserDirectoriesList();
await migrateGroupChatsMetadataFormat(directories);
await checkForNewContent(directories);
await ensureThumbnailCache(directories);
await diskCache.verify(directories);