diff --git a/Cargo.toml b/Cargo.toml index c251cf87..6709ee3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "core", "python", + "js", "jni", ] diff --git a/core/src/lib.rs b/core/src/lib.rs index 52cacb04..6477ba85 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -5,9 +5,13 @@ use fancy_regex::Regex; use rustc_hash::FxHashMap as HashMap; mod util; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] mod load; + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub mod openai_public; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] #[macro_use] extern crate lazy_static; diff --git a/js/Cargo.toml b/js/Cargo.toml index fac21a29..44d1081f 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -16,12 +16,9 @@ regex = "1.7.0" rustc-hash = "1.1.0" bstr = "1.0.1" wasm-bindgen = "0.2.83" -js-sys = "0.3.61" anyhow = "1.0.69" base64 = "0.21.0" gloo-utils = { version = "0.1", features = ["serde"] } -serde = { version = "1.0", features = ["derive"] } -reqwest = { version = "0.11.14", features = ["blocking"] } [features] default = ["inline"] diff --git a/js/scripts/override_any.ts b/js/scripts/override_any.ts index 0974e9b8..c07d1ae3 100644 --- a/js/scripts/override_any.ts +++ b/js/scripts/override_any.ts @@ -4,9 +4,9 @@ const project = new Project(); project.addSourceFilesAtPaths("./dist/**/*.ts"); for (const filename of [ - "./dist/bundler/_tiktoken.d.ts", - "./dist/node/_tiktoken.d.ts", - "./dist/web/_tiktoken.d.ts", + "./dist/bundler/_tiktoken_js.d.ts", + "./dist/node/_tiktoken_js.d.ts", + "./dist/web/_tiktoken_js.d.ts", ]) { const sourceFile = project.getSourceFileOrThrow(filename); const cls = sourceFile.getFirstDescendantByKindOrThrow( diff --git a/js/src/lib.rs b/js/src/lib.rs index 62345196..c575002b 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -245,11 +245,6 @@ impl Tiktoken { self.bpe.encode_single_token(&bytes).unwrap_throw() } - #[wasm_bindgen(skip_typescript)] - pub fn _encode_single_piece(&self, bytes: &[u8]) -> Vec { - self.bpe.encode_single_piece(&bytes) - } - pub fn decode(&self, tokens: Vec) -> Vec { self.bpe._decode_native(&tokens) }