Use a proper tokenizer for GPT-4o

This commit is contained in:
Cohee
2024-05-14 15:30:11 +03:00
parent 49cb8daf7d
commit 3113109f0a
3 changed files with 13 additions and 8 deletions
+1 -1
View File
@@ -2,7 +2,6 @@
"dependencies": {
"@agnai/sentencepiece-js": "^1.1.1",
"@agnai/web-tokenizers": "^0.1.3",
"@dqbd/tiktoken": "^1.0.13",
"@zeldafan0225/ai_horde": "^4.0.1",
"archiver": "^7.0.1",
"bing-translate-api": "^2.9.1",
@@ -36,6 +35,7 @@
"sanitize-filename": "^1.6.3",
"sillytavern-transformers": "^2.14.6",
"simple-git": "^3.19.1",
"tiktoken": "^1.0.15",
"vectra": "^0.2.2",
"wavefile": "^11.0.0",
"write-file-atomic": "^5.0.1",