@@ -10,7 +10,7 @@ import { ModelNotFoundError, UnsupportedTokenizerFilesError } from "./error";
1010import { Tokenizer } from "@mlc-ai/web-tokenizers" ;
1111import { ModelIntegrity , verifyIntegrity } from "./integrity" ;
1212
13- async function verifyTokenizerIfNeeded (
13+ async function maybeVerifyTokenizerIntegrity (
1414 data : ArrayBuffer ,
1515 filename : string ,
1616 url : string ,
@@ -147,7 +147,12 @@ export async function asyncLoadTokenizer(
147147 if ( config . tokenizer_files . includes ( "tokenizer.json" ) ) {
148148 const url = new URL ( "tokenizer.json" , baseUrl ) . href ;
149149 const model = await modelCache . fetchWithCache ( url , "arraybuffer" ) ;
150- await verifyTokenizerIfNeeded ( model , "tokenizer.json" , url , integrity ) ;
150+ await maybeVerifyTokenizerIntegrity (
151+ model ,
152+ "tokenizer.json" ,
153+ url ,
154+ integrity ,
155+ ) ;
151156 return Tokenizer . fromJSON ( model ) ;
152157 } else if ( config . tokenizer_files . includes ( "tokenizer.model" ) ) {
153158 logger (
@@ -159,7 +164,12 @@ export async function asyncLoadTokenizer(
159164 ) ;
160165 const url = new URL ( "tokenizer.model" , baseUrl ) . href ;
161166 const model = await modelCache . fetchWithCache ( url , "arraybuffer" ) ;
162- await verifyTokenizerIfNeeded ( model , "tokenizer.model" , url , integrity ) ;
167+ await maybeVerifyTokenizerIntegrity (
168+ model ,
169+ "tokenizer.model" ,
170+ url ,
171+ integrity ,
172+ ) ;
163173 return Tokenizer . fromSentencePiece ( model ) ;
164174 }
165175 throw new UnsupportedTokenizerFilesError ( config . tokenizer_files ) ;
0 commit comments