[lib]
crate-type = ["cdylib"]

[package]
name = "avante-tokenizers"
edition = { workspace = true }
version = { workspace = true }
rust-version = { workspace = true }
license = { workspace = true }

[lints]
workspace = true

[dependencies]
dirs = "5.0.1"
regex = "1.11.1"
hf-hub = { version = "0.3.2", features = ["default"] }
ureq = { version = "2.10.1", features = ["json", "socks-proxy"] }
mlua = { workspace = true }
tiktoken-rs = { workspace = true }
tokenizers = { workspace = true }

[features]
lua51 = ["mlua/lua51"]
lua52 = ["mlua/lua52"]
lua53 = ["mlua/lua53"]
lua54 = ["mlua/lua54"]
luajit = ["mlua/luajit"]