[lib]
crate-type = ["cdylib"]

[package]
name = "avante-tokenizers"
edition = { workspace = true }
version = { workspace = true }
rust-version = { workspace = true }
license = { workspace = true }

[lints]
workspace = true

[dependencies]
mlua = { workspace = true }
tiktoken-rs = { workspace = true }
tokenizers = { workspace = true }

[features]
lua51 = ["mlua/lua51"]
lua52 = ["mlua/lua52"]
lua53 = ["mlua/lua53"]
lua54 = ["mlua/lua54"]
luajit = ["mlua/luajit"]