Skip to content

Commit

Permalink
Sync codebase
Browse files Browse the repository at this point in the history
  • Loading branch information
hauntsaninja committed Apr 24, 2023
1 parent 46287bf commit f19feec
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
5 changes: 4 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,10 @@ impl CoreBPE {
let decoder: HashMap<usize, Vec<u8>> =
encoder.iter().map(|(k, v)| (*v, k.clone())).collect();

assert!(encoder.len() == decoder.len());
assert!(
encoder.len() == decoder.len(),
"Encoder and decoder must be of equal length; maybe you had duplicate token indices in your encoder?"
);

let special_tokens_decoder: HashMap<usize, Vec<u8>> = special_tokens_encoder
.iter()
Expand Down
4 changes: 3 additions & 1 deletion tiktoken/load.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ def read_file(blobpath: str) -> bytes:
with blobfile.BlobFile(blobpath, "rb") as f:
return f.read()
# avoiding blobfile for public files helps avoid auth issues, like MFA prompts
return requests.get(blobpath).content
resp = requests.get(blobpath)
resp.raise_for_status()
return resp.content


def read_file_cached(blobpath: str) -> bytes:
Expand Down

0 comments on commit f19feec

Please sign in to comment.