0557deeab7
* feat: tokenizers This reverts commit d5a4db8321d232a1b9c0d86fc38e8dd516d15776. * fix(inputs): #422 Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
33 lines
665 B
TOML
33 lines
665 B
TOML
[lib]
|
|
crate-type = ["cdylib"]
|
|
|
|
[package]
|
|
name = "avante-tokenizers"
|
|
edition = { workspace = true }
|
|
version = { workspace = true }
|
|
rust-version = { workspace = true }
|
|
license = { workspace = true }
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
mlua = { version = "0.10.0-beta.1", features = [
|
|
"module",
|
|
"serialize",
|
|
], git = "https://github.com/mlua-rs/mlua.git", branch = "main" }
|
|
tiktoken-rs = "0.5.9"
|
|
tokenizers = { version = "0.20.0", features = [
|
|
"esaxx_fast",
|
|
"http",
|
|
"unstable_wasm",
|
|
"onig",
|
|
], default-features = false }
|
|
|
|
[features]
|
|
lua51 = ["mlua/lua51"]
|
|
lua52 = ["mlua/lua52"]
|
|
lua53 = ["mlua/lua53"]
|
|
lua54 = ["mlua/lua54"]
|
|
luajit = ["mlua/luajit"]
|