Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix encoding if error

Browse Source
This commit is contained in:
rafaelsideguide 2024-10-18 11:50:58 -03:00
parent 6ebfcc85cf
commit aed11e72a6
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/lib/LLM-extraction/helpers.ts
View File

@ -6,7 +6,13 @@ export function numTokensFromString(message: string, model: string): number {
const encoder = encoding_for_model(model as TiktokenModel); const encoder = encoding_for_model(model as TiktokenModel);
// Encode the message into tokens // Encode the message into tokens
const tokens = encoder.encode(message); let tokens: Uint32Array;
try {
tokens = encoder.encode(message);
} catch (error) {
message = message.replace("<|endoftext|>", "");
tokens = encoder.encode(message);
}
// Free the encoder resources after use // Free the encoder resources after use
encoder.free(); encoder.free();