@@ -330,6 +330,43 @@ bool VocabStorage::const_iterator::operator!=(
330
330
return !(*this == Other);
331
331
}
332
332
333
+ Error VocabStorage::parseVocabSection (StringRef Key,
334
+ const json::Value &ParsedVocabValue,
335
+ VocabMap &TargetVocab, unsigned &Dim) {
336
+ json::Path::Root Path (" " );
337
+ const json::Object *RootObj = ParsedVocabValue.getAsObject ();
338
+ if (!RootObj)
339
+ return createStringError (errc::invalid_argument,
340
+ " JSON root is not an object" );
341
+
342
+ const json::Value *SectionValue = RootObj->get (Key);
343
+ if (!SectionValue)
344
+ return createStringError (errc::invalid_argument,
345
+ " Missing '" + std::string (Key) +
346
+ " ' section in vocabulary file" );
347
+ if (!json::fromJSON (*SectionValue, TargetVocab, Path))
348
+ return createStringError (errc::illegal_byte_sequence,
349
+ " Unable to parse '" + std::string (Key) +
350
+ " ' section from vocabulary" );
351
+
352
+ Dim = TargetVocab.begin ()->second .size ();
353
+ if (Dim == 0 )
354
+ return createStringError (errc::illegal_byte_sequence,
355
+ " Dimension of '" + std::string (Key) +
356
+ " ' section of the vocabulary is zero" );
357
+
358
+ if (!std::all_of (TargetVocab.begin (), TargetVocab.end (),
359
+ [Dim](const std::pair<StringRef, Embedding> &Entry) {
360
+ return Entry.second .size () == Dim;
361
+ }))
362
+ return createStringError (
363
+ errc::illegal_byte_sequence,
364
+ " All vectors in the '" + std::string (Key) +
365
+ " ' section of the vocabulary are not of the same dimension" );
366
+
367
+ return Error::success ();
368
+ }
369
+
333
370
// ==----------------------------------------------------------------------===//
334
371
// Vocabulary
335
372
// ===----------------------------------------------------------------------===//
@@ -460,43 +497,6 @@ VocabStorage Vocabulary::createDummyVocabForTest(unsigned Dim) {
460
497
// IR2VecVocabAnalysis
461
498
// ===----------------------------------------------------------------------===//
462
499
463
- Error IR2VecVocabAnalysis::parseVocabSection (
464
- StringRef Key, const json::Value &ParsedVocabValue, VocabMap &TargetVocab,
465
- unsigned &Dim) {
466
- json::Path::Root Path (" " );
467
- const json::Object *RootObj = ParsedVocabValue.getAsObject ();
468
- if (!RootObj)
469
- return createStringError (errc::invalid_argument,
470
- " JSON root is not an object" );
471
-
472
- const json::Value *SectionValue = RootObj->get (Key);
473
- if (!SectionValue)
474
- return createStringError (errc::invalid_argument,
475
- " Missing '" + std::string (Key) +
476
- " ' section in vocabulary file" );
477
- if (!json::fromJSON (*SectionValue, TargetVocab, Path))
478
- return createStringError (errc::illegal_byte_sequence,
479
- " Unable to parse '" + std::string (Key) +
480
- " ' section from vocabulary" );
481
-
482
- Dim = TargetVocab.begin ()->second .size ();
483
- if (Dim == 0 )
484
- return createStringError (errc::illegal_byte_sequence,
485
- " Dimension of '" + std::string (Key) +
486
- " ' section of the vocabulary is zero" );
487
-
488
- if (!std::all_of (TargetVocab.begin (), TargetVocab.end (),
489
- [Dim](const std::pair<StringRef, Embedding> &Entry) {
490
- return Entry.second .size () == Dim;
491
- }))
492
- return createStringError (
493
- errc::illegal_byte_sequence,
494
- " All vectors in the '" + std::string (Key) +
495
- " ' section of the vocabulary are not of the same dimension" );
496
-
497
- return Error::success ();
498
- }
499
-
500
500
// FIXME: Make this optional. We can avoid file reads
501
501
// by auto-generating a default vocabulary during the build time.
502
502
Error IR2VecVocabAnalysis::readVocabulary (VocabMap &OpcVocab,
@@ -513,16 +513,16 @@ Error IR2VecVocabAnalysis::readVocabulary(VocabMap &OpcVocab,
513
513
return ParsedVocabValue.takeError ();
514
514
515
515
unsigned OpcodeDim = 0 , TypeDim = 0 , ArgDim = 0 ;
516
- if (auto Err =
517
- parseVocabSection ( " Opcodes " , *ParsedVocabValue, OpcVocab, OpcodeDim))
516
+ if (auto Err = VocabStorage::parseVocabSection ( " Opcodes " , *ParsedVocabValue,
517
+ OpcVocab, OpcodeDim))
518
518
return Err;
519
519
520
- if (auto Err =
521
- parseVocabSection ( " Types " , *ParsedVocabValue, TypeVocab, TypeDim))
520
+ if (auto Err = VocabStorage::parseVocabSection ( " Types " , *ParsedVocabValue,
521
+ TypeVocab, TypeDim))
522
522
return Err;
523
523
524
- if (auto Err =
525
- parseVocabSection ( " Arguments " , *ParsedVocabValue, ArgVocab, ArgDim))
524
+ if (auto Err = VocabStorage::parseVocabSection ( " Arguments " , *ParsedVocabValue,
525
+ ArgVocab, ArgDim))
526
526
return Err;
527
527
528
528
if (!(OpcodeDim == TypeDim && TypeDim == ArgDim))
0 commit comments