Tiktoken Calculate Tokens

Tiktoken Calculate Tokens

import sys
import tiktoken

def count_tokens(filename: str, model_name="gpt-4") -> int:
"""Count the number of tokens in a file using TikToken."""
try:
with open(filename, 'r') as file:
content = file.read()
# Get the tokenizer encoding for the specified model
encoding = tiktoken.encoding_for_model(model_name)
tokens = encoding.encode(content)
return len(tokens)
except FileNotFoundError:
print("File not found.")
return 0

if name == "main":
if len(sys.argv) != 2:
print("Usage: python script.py ")
else:
filename = sys.argv[1]
print("Number of tokens:", count_tokens(filename))

Usage

python script.py [FILENAME]