add github action

.github/workflows/build_wheels.yml

@@ -51,3 +51,52 @@ jobs:
  with:
  name: dist
  path: ./dist/*.tar.gz
+
+ build_jni:
+ name: jni on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ include:
+ - os: ubuntu-latest
+ outdir: linux_64
+ - os: windows-latest
+ outdir: windows_64
+ - os: macos-latest
+ outdir: osx_64
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Install rust toolchain
+ uses: actions-rs/toolchain@v1
+ with:
+ # stable doesn't have --out-dir
+ toolchain: nightly
+
+ - name: Build
+ working-directory: ./jni
+ # TODO: 32bit vs 64bit?
+ # https://github.com/scijava/native-lib-loader
+ run: cargo build --release -Z unstable-options --out-dir ../build/natives/${{ env.outdir }}
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: natives
+ path: ./build/*
+
+ build_java:
+ name: java
+ runs-on: ubuntu-latest
+ needs: [build_jni]
+
+ steps:
+ - name: Load outputs
+ uses: actions/download-artifact@v3
+ with:
+ name: natives
+ path: natives
+
+ - name: debug
+ run: find natives

Cargo.toml

@@ -4,4 +4,12 @@ members = [
  "core",
  "python",
  "jni",
-]
+]
+
+[profile.release]
+incremental = true
+opt-level = 'z' # Optimize for size
+lto = true # Enable link-time optimization
+codegen-units = 1 # Reduce number of codegen units to increase optimizations
+panic = 'abort' # Abort on panic
+strip = true # Strip symbols from binary*

core/Cargo.toml

@@ -19,6 +19,3 @@ sha1 = "0.10.5"
 json = "0.12.4"
 base64 = "0.21.0"
 lazy_static = "1.4.0"
-
-[profile.release]
-incremental = true

jni/Cargo.toml

@@ -13,10 +13,3 @@ tiktoken_core = { path = "../core" }
 rustc-hash = "1.1.0"
 jni = "0.20.0"
 
-[profile.release]
-incremental = true
-opt-level = 'z' # Optimize for size
-lto = true # Enable link-time optimization
-codegen-units = 1 # Reduce number of codegen units to increase optimizations
-panic = 'abort' # Abort on panic
-strip = true # Strip symbols from binary*

python/Cargo.toml

@@ -12,6 +12,3 @@ crate-type = ["cdylib"]
 pyo3 = { version = "0.17.3", features = ["extension-module"] }
 tiktoken_core = { path = "../core" }
 rustc-hash = "1.1.0"
-
-[profile.release]
-incremental = true