[tiktoken] hello world

.gitignore

@@ -0,0 +1,42 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Environments
+.env
+.venv
+
+# Tools
+.mypy_cache
+.coverage
+htmlcov
+
+# General
+.DS_Store
+
+Cargo.lock
+target/

Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "tiktoken"
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.57.0"
+
+[lib]
+name = "_tiktoken"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.17.3", features = ["extension-module"] }
+
+# tiktoken dependencies
+fancy-regex = "0.10.0"
+regex = "1.7.0"
+rustc-hash = "1.1.0"
+bstr = "1.0.1"
+
+[profile.release]
+incremental = true

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 OpenAI, Shantanu Jain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,5 @@
+include *.svg
+include *.toml
+include Makefile
+recursive-include scripts *.py
+recursive-include src *.rs

Makefile

@@ -0,0 +1,49 @@
+PROJECT := tiktoken
+
+.PHONY: default
+default: editable_install
+
+.PHONY: install_rust
+install_rust:
+ which cargo >/dev/null || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.62
+
+.PHONY: clean
+clean:
+ cargo clean
+ pip uninstall -y $(PROJECT)
+ find . | grep -E '__pycache__|\.pyc' | xargs rm -rf
+ find . | grep -E '\.so' | xargs rm -rf
+ rm -rf dist/ build/
+ rm -rf $(PROJECT).egg-info/
+
+.PHONY: format
+format:
+ @ which black >/dev/null || python3 -m pip install black
+ @ which isort >/dev/null || python3 -m pip install isort
+ cargo fmt -- --config group_imports=StdExternalCrate
+ black --line-length 100 --skip-magic-trailing-comma --quiet .
+ isort --line-length 100 --profile black --quiet .
+
+
+.PHONY: format_check
+format_check:
+ @ which black >/dev/null || python3 -m pip install black
+ @ which isort >/dev/null || python3 -m pip install isort
+ cargo fmt --check -- --config group_imports=StdExternalCrate
+ black --check --line-length 100 --skip-magic-trailing-comma --quiet .
+ isort --check --line-length 100 --profile black --quiet .
+
+.PHONY: lint
+lint:
+ cargo clippy --all -- -D warnings
+ @ which flake8 >/dev/null || python3 -m pip install flake8==5 flake8-bugbear==22.9.11
+ flake8 --ignore=E203,E501,W503,E731 --per-file-ignores="$(PROJECT)/__init__.py:F401 setup.py:E402" --exclude=build .
+
+.PHONY: editable_install
+editable_install:
+ @ if [ -f $(PROJECT).egg-info ]; then \
+ pip install --disable-pip-version-check --progress-bar=off setuptools wheel setuptools-rust ; \
+ pip install --disable-pip-version-check --no-build-isolation -e . ; \
+ else \
+ pip install --disable-pip-version-check --no-deps --no-build-isolation --ignore-installed -e . ; \
+ fi

README.md

@@ -0,0 +1,28 @@
+# ⏳ tiktoken
+
+tiktoken is a fast tokeniser.
+
+```python
+import tiktoken
+enc = tiktoken.get_encoding("gpt2")
+print(enc.encode("hello world"))
+```
+
+The open source version of `tiktoken` can be installed from PyPI:
+```
+pip install tiktoken
+```
+
+The tokeniser API is documented in `tiktoken/core.py`.
+
+
+## Performance
+
+`tiktoken` is between 3-6x faster than huggingface's tokeniser:
+
+![image](./perf.svg)
+
+Performance measured on 1GB of text using the GPT-2 tokeniser, using `GPT2TokenizerFast` from
+`tokenizers==0.13.2` and `transformers==4.24.0`.
+
+