Sync codebase

.github/workflows/build_wheels.yml

@@ -16,12 +16,12 @@ jobs:
  # cibuildwheel builds linux wheels inside a manylinux container
  # it also takes care of procuring the correct python version for us
  os: [ubuntu-latest, windows-latest, macos-latest]
- python-version: [38, 39, 310, 311]
+ python-version: [38, 39, 310, 311, 312]
 
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
- - uses: pypa/cibuildwheel@v2.11.3
+ - uses: pypa/cibuildwheel@v2.16.2
  env:
  CIBW_BUILD: "cp${{ matrix.python-version}}-*"
 
@@ -37,18 +37,18 @@ jobs:
  fail-fast: false
  matrix:
  os: [ubuntu-latest]
- python-version: [38, 39, 310, 311]
+ python-version: [38, 39, 310, 311, 312]
 
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
  - name: Setup up QEMU
- uses: docker/setup-qemu-action@v2
+ uses: docker/setup-qemu-action@v3
  with:
  platforms: arm64
 
  - name: Build wheels
- uses: pypa/cibuildwheel@v2.11.3
+ uses: pypa/cibuildwheel@v2.16.2
  env:
  CIBW_BUILD: "cp${{ matrix.python-version}}-*"
  CIBW_ARCHS: aarch64
@@ -64,7 +64,7 @@ jobs:
  name: sdist
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - uses: actions/setup-python@v4
  name: Install Python
  with:

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 This is the changelog for the open source version of tiktoken.
 
+## [v0.5.2]
+- Build wheels for Python 3.12
+- Update version of PyO3 to allow multiple imports
+
 ## [v0.5.1]
 - Add `encoding_name_for_model`, undo some renames to variables that are implementation details
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tiktoken"
-version = "0.5.1"
+version = "0.5.2"
 edition = "2021"
 rust-version = "1.57.0"
 
@@ -9,7 +9,7 @@ name = "_tiktoken"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.19.0", features = ["extension-module"] }
+pyo3 = { version = "0.20.0", features = ["extension-module"] }
 
 # tiktoken dependencies
 fancy-regex = "0.11.0"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tiktoken"
-version = "0.5.1"
+version = "0.5.2"
 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -42,3 +42,4 @@ test-command = "pytest {project}/tests --import-mode=append"
 [[tool.cibuildwheel.overrides]]
 select = "*linux_aarch64"
 test-command = """python -c 'import tiktoken; enc = tiktoken.get_encoding("gpt2"); assert enc.encode("hello world") == [31373, 995]'"""
+