Use a proper tokenizer for GPT-4o
This commit is contained in:
+1
-1
@@ -2,7 +2,6 @@
|
||||
"dependencies": {
|
||||
"@agnai/sentencepiece-js": "^1.1.1",
|
||||
"@agnai/web-tokenizers": "^0.1.3",
|
||||
"@dqbd/tiktoken": "^1.0.13",
|
||||
"@zeldafan0225/ai_horde": "^4.0.1",
|
||||
"archiver": "^7.0.1",
|
||||
"bing-translate-api": "^2.9.1",
|
||||
@@ -36,6 +35,7 @@
|
||||
"sanitize-filename": "^1.6.3",
|
||||
"sillytavern-transformers": "^2.14.6",
|
||||
"simple-git": "^3.19.1",
|
||||
"tiktoken": "^1.0.15",
|
||||
"vectra": "^0.2.2",
|
||||
"wavefile": "^11.0.0",
|
||||
"write-file-atomic": "^5.0.1",
|
||||
|
||||
Reference in New Issue
Block a user