[package]
name = "tokenizers"
version = { workspace = true }
edition = { workspace = true }

[features]
icu = ["rust_icu_ubrk", "rust_icu_sys", "rust_icu_uloc", "rust_icu_ustring"]

[dependencies]
anyhow = "1.0.100"
lindera = { version = "1.4.1", features = [
  "embedded-cc-cedict",
  "embedded-ipadic",
  "embedded-ko-dic",
  "compress",
] }
once_cell = "1.21.3"
serde = "1.0.228"
serde_json = "1.0.145"
tantivy.workspace = true
tracing = "0.1.43"
strum_macros = "0.27.2"
strum = { version = "0.27.2", features = ["derive"] }
tantivy-jieba = { workspace = true }
emoji = "0.2.1"
unicode-segmentation = "1.12.0"
opencc-jieba-rs = "0.7.2"

[dependencies.rust_icu_ubrk]
version = "5.4.0"
optional = true

[dependencies.rust_icu_sys]
version = "5.4.0"
optional = true

[dependencies.rust_icu_ustring]
version = "5.4.0"
optional = true

[dependencies.rust_icu_uloc]
version = "5.4.0"
optional = true

[dev-dependencies]
rstest = "0.25.0"

[dependencies.rust_icu_common]
version = "5.4.0"
optional = true

[package.metadata.cargo-machete]
ignored = ["rust_icu_common", "strum"]