Make sure JS builds

Cargo.toml

@@ -3,6 +3,7 @@
 members = [
  "core",
  "python",
+ "js",
  "jni",
 ]
 

core/src/lib.rs

@@ -5,9 +5,13 @@ use fancy_regex::Regex;
 use rustc_hash::FxHashMap as HashMap;
 
 mod util;
+#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
 mod load;
+
+#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
 pub mod openai_public;
 
+#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
 #[macro_use]
 extern crate lazy_static;
 

js/Cargo.toml

@@ -16,12 +16,9 @@ regex = "1.7.0"
 rustc-hash = "1.1.0"
 bstr = "1.0.1"
 wasm-bindgen = "0.2.83"
-js-sys = "0.3.61"
 anyhow = "1.0.69"
 base64 = "0.21.0"
 gloo-utils = { version = "0.1", features = ["serde"] }
-serde = { version = "1.0", features = ["derive"] }
-reqwest = { version = "0.11.14", features = ["blocking"] }
 
 [features]
 default = ["inline"]

js/scripts/override_any.ts

@@ -4,9 +4,9 @@ const project = new Project();
 project.addSourceFilesAtPaths("./dist/**/*.ts");
 
 for (const filename of [
- "./dist/bundler/_tiktoken.d.ts",
- "./dist/node/_tiktoken.d.ts",
- "./dist/web/_tiktoken.d.ts",
+ "./dist/bundler/_tiktoken_js.d.ts",
+ "./dist/node/_tiktoken_js.d.ts",
+ "./dist/web/_tiktoken_js.d.ts",
 ]) {
  const sourceFile = project.getSourceFileOrThrow(filename);
  const cls = sourceFile.getFirstDescendantByKindOrThrow(

js/src/lib.rs

@@ -245,11 +245,6 @@ impl Tiktoken {
  self.bpe.encode_single_token(&bytes).unwrap_throw()
  }
 
- #[wasm_bindgen(skip_typescript)]
- pub fn _encode_single_piece(&self, bytes: &[u8]) -> Vec<usize> {
- self.bpe.encode_single_piece(&bytes)
- }
-
  pub fn decode(&self, tokens: Vec<usize>) -> Vec<u8> {
  self.bpe._decode_native(&tokens)
  }