mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 11:42:24 +08:00
17 lines
534 B
TypeScript
17 lines
534 B
TypeScript
|
import { encoding_for_model } from "@dqbd/tiktoken";
|
||
|
|
import { TiktokenModel } from "@dqbd/tiktoken";
|
||
|
|
|
||
|
|
// This function calculates the number of tokens in a text string using GPT-3.5-turbo model
|
||
|
|
export function numTokensFromString(message: string, model: string): number {
|
||
|
|
const encoder = encoding_for_model(model as TiktokenModel);
|
||
|
|
|
||
|
|
// Encode the message into tokens
|
||
|
|
const tokens = encoder.encode(message);
|
||
|
|
|
||
|
|
// Free the encoder resources after use
|
||
|
|
encoder.free();
|
||
|
|
|
||
|
|
// Return the number of tokens
|
||
|
|
return tokens.length;
|
||
|
|
}
|