d2095ba267
* feat: autobuild tiktoken lib and schenanigans Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: revert readme changes Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(build): windows Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz> * chore(plugin): early load commands and base setup Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(build): make sync Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * feat: rust go vroom vroom Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * feat: scuffed afaf implementation binding go brrrr Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: remove dups Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(tokens): calculate whether we should do prompt_caching (fixes #416) Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: ignore lockfiles Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * Update README.md * Update crates/avante-tokenizers/README.md * chore: remove unused Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: remove auto build Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz> Co-authored-by: yuchanns <me@yuchanns.xyz>
33 lines
665 B
TOML
33 lines
665 B
TOML
[lib]
|
|
crate-type = ["cdylib"]
|
|
|
|
[package]
|
|
name = "avante-tokenizers"
|
|
edition = { workspace = true }
|
|
version = { workspace = true }
|
|
rust-version = { workspace = true }
|
|
license = { workspace = true }
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
mlua = { version = "0.10.0-beta.1", features = [
|
|
"module",
|
|
"serialize",
|
|
], git = "https://github.com/mlua-rs/mlua.git", branch = "main" }
|
|
tiktoken-rs = "0.5.9"
|
|
tokenizers = { version = "0.20.0", features = [
|
|
"esaxx_fast",
|
|
"http",
|
|
"unstable_wasm",
|
|
"onig",
|
|
], default-features = false }
|
|
|
|
[features]
|
|
lua51 = ["mlua/lua51"]
|
|
lua52 = ["mlua/lua52"]
|
|
lua53 = ["mlua/lua53"]
|
|
lua54 = ["mlua/lua54"]
|
|
luajit = ["mlua/luajit"]
|