From 3f3e68c492fa772773a2b1c9a1ad79aaf35b7d81 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 8 Jan 2024 00:46:54 +0200 Subject: [PATCH 1/6] document some functions --- public/scripts/extensions/expressions/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 3ceaca866..f174b3c90 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -385,6 +385,9 @@ function onExpressionsShowDefaultInput() { } } +/** + * Stops animating a talkinghead. + */ async function unloadTalkingHead() { if (!modules.includes('talkinghead')) { console.debug('talkinghead module is disabled'); @@ -404,6 +407,9 @@ async function unloadTalkingHead() { } } +/** + * Posts `talkinghead.png` of the current character to the talkinghead module in SillyTavern-extras, to start animating it. + */ async function loadTalkingHead() { if (!modules.includes('talkinghead')) { console.debug('talkinghead module is disabled'); From 1184ea2c3b4c621f0b6c9cc76797be94da4adb22 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 8 Jan 2024 00:47:18 +0200 Subject: [PATCH 2/6] clarity --- public/scripts/extensions/expressions/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index f174b3c90..2add4dd81 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -569,9 +569,11 @@ async function moduleWorker() { return; } + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) + && lastMessage === currentLastMessage.mes); + // check if last message changed - if ((lastCharacter === context.characterId || lastCharacter === context.groupId) - && lastMessage === currentLastMessage.mes) { + if (!lastMessageChanged) { return; } From 39c485ab2588a0fcfad88bd7d80a82900e742550 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 8 Jan 2024 00:47:40 +0200 Subject: [PATCH 3/6] some missed refactorings --- public/scripts/extensions/expressions/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 2add4dd81..5c1141c5d 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -1019,7 +1019,7 @@ async function getExpressionsList() { } async function setExpression(character, expression, force) { - if (extension_settings.expressions.local || !extension_settings.expressions.talkinghead) { + if (!isTalkingHeadEnabled()) { console.debug('entered setExpressions'); await validateImages(character); const img = $('img.expression'); @@ -1284,7 +1284,7 @@ async function onClickExpressionUpload(event) { e.target.form.reset(); // In talkinghead mode, when a new talkinghead image is uploaded, refresh the live char. - if (extension_settings.expressions.talkinghead && !extension_settings.expressions.local && id === 'talkinghead') { + if (isTalkingHeadEnabled() && id === 'talkinghead') { await loadTalkingHead(); } }; From 87b05e2e2da2b25b8763fa00b6005cabef030b8a Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 8 Jan 2024 00:47:59 +0200 Subject: [PATCH 4/6] enable talkinghead's talking animation while the LLM is streaming --- .../scripts/extensions/expressions/index.js | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 5c1141c5d..c028c2a5f 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -10,6 +10,7 @@ export { MODULE_NAME }; const MODULE_NAME = 'expressions'; const UPDATE_INTERVAL = 2000; const STREAMING_UPDATE_INTERVAL = 6000; +const TALKINGCHECK_UPDATE_INTERVAL = 250; const FALLBACK_EXPRESSION = 'joy'; const DEFAULT_EXPRESSIONS = [ 'talkinghead', @@ -46,6 +47,8 @@ const DEFAULT_EXPRESSIONS = [ let expressionsList = null; let lastCharacter = undefined; let lastMessage = null; +let lastTalkingState = false; +let lastTalkingStateMessage = null; // last message as seen by `updateTalkingState` (tracked separately, different timer) let spriteCache = {}; let inApiCall = false; let lastServerResponseTime = 0; @@ -623,6 +626,62 @@ async function moduleWorker() { } } +/** + * Starts/stops talkinghead talking animation. + * + * Talking starts only when all the following conditions are met: + * - The LLM is currently streaming its output. + * - The AI's current last message is non-empty, and also not just '...' (as produced by a swipe). + * - The AI's current last message has changed from what we saw during the previous call. + * + * In all other cases, talking stops. + * + * A talkinghead API call is made only when the talking state changes. + */ +async function updateTalkingState() { + const context = getContext(); + const currentLastMessage = getLastCharacterMessage(); + + try { + // TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker` + // (that similarly checks the streaming processor state) does that for some reason. + // Talkinghead isn't currently designed to work with groups. + if (isTalkingHeadEnabled()) { + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) + && lastTalkingStateMessage === currentLastMessage.mes); + const url = new URL(getApiUrl()); + let newTalkingState; + if (context.streamingProcessor && !context.streamingProcessor.isFinished && + currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) { + url.pathname = '/api/talkinghead/start_talking'; + newTalkingState = true; + } else { + url.pathname = '/api/talkinghead/stop_talking'; + newTalkingState = false; + } + try { + // Call the talkinghead API only if the talking state changed. + if (newTalkingState !== lastTalkingState) { + console.debug(`updateTalkingState: calling ${url.pathname}`); + await doExtrasFetch(url); + } + } + catch (error) { + // it's ok if not supported + } + finally { + lastTalkingState = newTalkingState; + } + } + } + catch (error) { + // console.log(error); + } + finally { + lastTalkingStateMessage = currentLastMessage.mes; + } +} + /** * Checks whether the current character has a talkinghead image available. * @returns {Promise} True if the character has a talkinghead image available, false otherwise. @@ -1513,6 +1572,11 @@ function setExpressionOverrideHtml(forceClear = false) { const updateFunction = wrapper.update.bind(wrapper); setInterval(updateFunction, UPDATE_INTERVAL); moduleWorker(); + // For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule. + const wrapper_talkingstate = new ModuleWorkerWrapper(updateTalkingState); + const updateTalkingStateFunction = wrapper_talkingstate.update.bind(wrapper_talkingstate); + setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL); + updateTalkingState(); dragElement($('#expression-holder')); eventSource.on(event_types.CHAT_CHANGED, () => { // character changed From 63ab16161fbf69f2cd4d74b2ae010943bd2ac75b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 9 Jan 2024 19:52:49 +0200 Subject: [PATCH 5/6] Add check for enabled module --- .../scripts/extensions/expressions/index.js | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index c028c2a5f..55153f372 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -572,8 +572,7 @@ async function moduleWorker() { return; } - const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) - && lastMessage === currentLastMessage.mes); + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastMessage === currentLastMessage.mes); // check if last message changed if (!lastMessageChanged) { @@ -639,6 +638,11 @@ async function moduleWorker() { * A talkinghead API call is made only when the talking state changes. */ async function updateTalkingState() { + // Don't bother if talkinghead is disabled or not loaded. + if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) { + return; + } + const context = getContext(); const currentLastMessage = getLastCharacterMessage(); @@ -646,33 +650,30 @@ async function updateTalkingState() { // TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker` // (that similarly checks the streaming processor state) does that for some reason. // Talkinghead isn't currently designed to work with groups. - if (isTalkingHeadEnabled()) { - const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) - && lastTalkingStateMessage === currentLastMessage.mes); - const url = new URL(getApiUrl()); - let newTalkingState; - if (context.streamingProcessor && !context.streamingProcessor.isFinished && - currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) { - url.pathname = '/api/talkinghead/start_talking'; - newTalkingState = true; - } else { - url.pathname = '/api/talkinghead/stop_talking'; - newTalkingState = false; - } - try { - // Call the talkinghead API only if the talking state changed. - if (newTalkingState !== lastTalkingState) { - console.debug(`updateTalkingState: calling ${url.pathname}`); - await doExtrasFetch(url); - } - } - catch (error) { - // it's ok if not supported - } - finally { - lastTalkingState = newTalkingState; + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastTalkingStateMessage === currentLastMessage.mes); + const url = new URL(getApiUrl()); + let newTalkingState; + if (context.streamingProcessor && !context.streamingProcessor.isFinished && + currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) { + url.pathname = '/api/talkinghead/start_talking'; + newTalkingState = true; + } else { + url.pathname = '/api/talkinghead/stop_talking'; + newTalkingState = false; + } + try { + // Call the talkinghead API only if the talking state changed. + if (newTalkingState !== lastTalkingState) { + console.debug(`updateTalkingState: calling ${url.pathname}`); + await doExtrasFetch(url); } } + catch (error) { + // it's ok if not supported + } + finally { + lastTalkingState = newTalkingState; + } } catch (error) { // console.log(error); @@ -1573,8 +1574,8 @@ function setExpressionOverrideHtml(forceClear = false) { setInterval(updateFunction, UPDATE_INTERVAL); moduleWorker(); // For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule. - const wrapper_talkingstate = new ModuleWorkerWrapper(updateTalkingState); - const updateTalkingStateFunction = wrapper_talkingstate.update.bind(wrapper_talkingstate); + const wrapperTalkingState = new ModuleWorkerWrapper(updateTalkingState); + const updateTalkingStateFunction = wrapperTalkingState.update.bind(wrapperTalkingState); setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL); updateTalkingState(); dragElement($('#expression-holder')); From 810667e8e79cc15debd9f5260ec520a141f1ea12 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 9 Jan 2024 20:17:17 +0200 Subject: [PATCH 6/6] Slight increase processing delay --- public/scripts/extensions/expressions/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 55153f372..451577a43 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -10,7 +10,7 @@ export { MODULE_NAME }; const MODULE_NAME = 'expressions'; const UPDATE_INTERVAL = 2000; const STREAMING_UPDATE_INTERVAL = 6000; -const TALKINGCHECK_UPDATE_INTERVAL = 250; +const TALKINGCHECK_UPDATE_INTERVAL = 500; const FALLBACK_EXPRESSION = 'joy'; const DEFAULT_EXPRESSIONS = [ 'talkinghead',