diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 3ceaca866..451577a43 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -10,6 +10,7 @@ export { MODULE_NAME }; const MODULE_NAME = 'expressions'; const UPDATE_INTERVAL = 2000; const STREAMING_UPDATE_INTERVAL = 6000; +const TALKINGCHECK_UPDATE_INTERVAL = 500; const FALLBACK_EXPRESSION = 'joy'; const DEFAULT_EXPRESSIONS = [ 'talkinghead', @@ -46,6 +47,8 @@ const DEFAULT_EXPRESSIONS = [ let expressionsList = null; let lastCharacter = undefined; let lastMessage = null; +let lastTalkingState = false; +let lastTalkingStateMessage = null; // last message as seen by `updateTalkingState` (tracked separately, different timer) let spriteCache = {}; let inApiCall = false; let lastServerResponseTime = 0; @@ -385,6 +388,9 @@ function onExpressionsShowDefaultInput() { } } +/** + * Stops animating a talkinghead. + */ async function unloadTalkingHead() { if (!modules.includes('talkinghead')) { console.debug('talkinghead module is disabled'); @@ -404,6 +410,9 @@ async function unloadTalkingHead() { } } +/** + * Posts `talkinghead.png` of the current character to the talkinghead module in SillyTavern-extras, to start animating it. + */ async function loadTalkingHead() { if (!modules.includes('talkinghead')) { console.debug('talkinghead module is disabled'); @@ -563,9 +572,10 @@ async function moduleWorker() { return; } + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastMessage === currentLastMessage.mes); + // check if last message changed - if ((lastCharacter === context.characterId || lastCharacter === context.groupId) - && lastMessage === currentLastMessage.mes) { + if (!lastMessageChanged) { return; } @@ -615,6 +625,64 @@ async function moduleWorker() { } } +/** + * Starts/stops talkinghead talking animation. + * + * Talking starts only when all the following conditions are met: + * - The LLM is currently streaming its output. + * - The AI's current last message is non-empty, and also not just '...' (as produced by a swipe). + * - The AI's current last message has changed from what we saw during the previous call. + * + * In all other cases, talking stops. + * + * A talkinghead API call is made only when the talking state changes. + */ +async function updateTalkingState() { + // Don't bother if talkinghead is disabled or not loaded. + if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) { + return; + } + + const context = getContext(); + const currentLastMessage = getLastCharacterMessage(); + + try { + // TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker` + // (that similarly checks the streaming processor state) does that for some reason. + // Talkinghead isn't currently designed to work with groups. + const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastTalkingStateMessage === currentLastMessage.mes); + const url = new URL(getApiUrl()); + let newTalkingState; + if (context.streamingProcessor && !context.streamingProcessor.isFinished && + currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) { + url.pathname = '/api/talkinghead/start_talking'; + newTalkingState = true; + } else { + url.pathname = '/api/talkinghead/stop_talking'; + newTalkingState = false; + } + try { + // Call the talkinghead API only if the talking state changed. + if (newTalkingState !== lastTalkingState) { + console.debug(`updateTalkingState: calling ${url.pathname}`); + await doExtrasFetch(url); + } + } + catch (error) { + // it's ok if not supported + } + finally { + lastTalkingState = newTalkingState; + } + } + catch (error) { + // console.log(error); + } + finally { + lastTalkingStateMessage = currentLastMessage.mes; + } +} + /** * Checks whether the current character has a talkinghead image available. * @returns {Promise} True if the character has a talkinghead image available, false otherwise. @@ -1011,7 +1079,7 @@ async function getExpressionsList() { } async function setExpression(character, expression, force) { - if (extension_settings.expressions.local || !extension_settings.expressions.talkinghead) { + if (!isTalkingHeadEnabled()) { console.debug('entered setExpressions'); await validateImages(character); const img = $('img.expression'); @@ -1276,7 +1344,7 @@ async function onClickExpressionUpload(event) { e.target.form.reset(); // In talkinghead mode, when a new talkinghead image is uploaded, refresh the live char. - if (extension_settings.expressions.talkinghead && !extension_settings.expressions.local && id === 'talkinghead') { + if (isTalkingHeadEnabled() && id === 'talkinghead') { await loadTalkingHead(); } }; @@ -1505,6 +1573,11 @@ function setExpressionOverrideHtml(forceClear = false) { const updateFunction = wrapper.update.bind(wrapper); setInterval(updateFunction, UPDATE_INTERVAL); moduleWorker(); + // For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule. + const wrapperTalkingState = new ModuleWorkerWrapper(updateTalkingState); + const updateTalkingStateFunction = wrapperTalkingState.update.bind(wrapperTalkingState); + setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL); + updateTalkingState(); dragElement($('#expression-holder')); eventSource.on(event_types.CHAT_CHANGED, () => { // character changed