@@ -361,6 +361,74 @@ static void addVPMetadata(Module &M, Instruction &I,
361
361
}
362
362
}
363
363
364
+ static void
365
+ handleAllocSite (Instruction &I, CallBase *CI,
366
+ ArrayRef<uint64_t > InlinedCallStack, LLVMContext &Ctx,
367
+ OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize,
368
+ const std::set<const AllocationInfo *> &AllocInfoSet,
369
+ std::map<std::pair<uint64_t , unsigned >, AllocMatchInfo>
370
+ &FullStackIdToAllocMatchInfo) {
371
+ // We may match this instruction's location list to multiple MIB
372
+ // contexts. Add them to a Trie specialized for trimming the contexts to
373
+ // the minimal needed to disambiguate contexts with unique behavior.
374
+ CallStackTrie AllocTrie (&ORE, MaxColdSize);
375
+ uint64_t TotalSize = 0 ;
376
+ uint64_t TotalColdSize = 0 ;
377
+ for (auto *AllocInfo : AllocInfoSet) {
378
+ // Check the full inlined call stack against this one.
379
+ // If we found and thus matched all frames on the call, include
380
+ // this MIB.
381
+ if (stackFrameIncludesInlinedCallStack (AllocInfo->CallStack ,
382
+ InlinedCallStack)) {
383
+ NumOfMemProfMatchedAllocContexts++;
384
+ uint64_t FullStackId = 0 ;
385
+ if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis ())
386
+ FullStackId = computeFullStackId (AllocInfo->CallStack );
387
+ auto AllocType = addCallStack (AllocTrie, AllocInfo, FullStackId);
388
+ TotalSize += AllocInfo->Info .getTotalSize ();
389
+ if (AllocType == AllocationType::Cold)
390
+ TotalColdSize += AllocInfo->Info .getTotalSize ();
391
+ // Record information about the allocation if match info printing
392
+ // was requested.
393
+ if (ClPrintMemProfMatchInfo) {
394
+ assert (FullStackId != 0 );
395
+ FullStackIdToAllocMatchInfo[std::make_pair (FullStackId,
396
+ InlinedCallStack.size ())] = {
397
+ AllocInfo->Info .getTotalSize (), AllocType};
398
+ }
399
+ }
400
+ }
401
+ // If the threshold for the percent of cold bytes is less than 100%,
402
+ // and not all bytes are cold, see if we should still hint this
403
+ // allocation as cold without context sensitivity.
404
+ if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 &&
405
+ TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) {
406
+ AllocTrie.addSingleAllocTypeAttribute (CI, AllocationType::Cold, " dominant" );
407
+ return ;
408
+ }
409
+
410
+ // We might not have matched any to the full inlined call stack.
411
+ // But if we did, create and attach metadata, or a function attribute if
412
+ // all contexts have identical profiled behavior.
413
+ if (!AllocTrie.empty ()) {
414
+ NumOfMemProfMatchedAllocs++;
415
+ // MemprofMDAttached will be false if a function attribute was
416
+ // attached.
417
+ bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata (CI);
418
+ assert (MemprofMDAttached == I.hasMetadata (LLVMContext::MD_memprof));
419
+ if (MemprofMDAttached) {
420
+ // Add callsite metadata for the instruction's location list so that
421
+ // it simpler later on to identify which part of the MIB contexts
422
+ // are from this particular instruction (including during inlining,
423
+ // when the callsite metadata will be updated appropriately).
424
+ // FIXME: can this be changed to strip out the matching stack
425
+ // context ids from the MIB contexts and not add any callsite
426
+ // metadata here to save space?
427
+ addCallsiteMetadata (I, InlinedCallStack, Ctx);
428
+ }
429
+ }
430
+ }
431
+
364
432
static void readMemprof (Module &M, Function &F,
365
433
IndexedInstrProfReader *MemProfReader,
366
434
const TargetLibraryInfo &TLI,
@@ -554,66 +622,8 @@ static void readMemprof(Module &M, Function &F,
554
622
if (AllocInfoIter != LocHashToAllocInfo.end () &&
555
623
// Only consider allocations which support hinting.
556
624
isAllocationWithHotColdVariant (CI->getCalledFunction (), TLI)) {
557
- // We may match this instruction's location list to multiple MIB
558
- // contexts. Add them to a Trie specialized for trimming the contexts to
559
- // the minimal needed to disambiguate contexts with unique behavior.
560
- CallStackTrie AllocTrie (&ORE, MaxColdSize);
561
- uint64_t TotalSize = 0 ;
562
- uint64_t TotalColdSize = 0 ;
563
- for (auto *AllocInfo : AllocInfoIter->second ) {
564
- // Check the full inlined call stack against this one.
565
- // If we found and thus matched all frames on the call, include
566
- // this MIB.
567
- if (stackFrameIncludesInlinedCallStack (AllocInfo->CallStack ,
568
- InlinedCallStack)) {
569
- NumOfMemProfMatchedAllocContexts++;
570
- uint64_t FullStackId = 0 ;
571
- if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis ())
572
- FullStackId = computeFullStackId (AllocInfo->CallStack );
573
- auto AllocType = addCallStack (AllocTrie, AllocInfo, FullStackId);
574
- TotalSize += AllocInfo->Info .getTotalSize ();
575
- if (AllocType == AllocationType::Cold)
576
- TotalColdSize += AllocInfo->Info .getTotalSize ();
577
- // Record information about the allocation if match info printing
578
- // was requested.
579
- if (ClPrintMemProfMatchInfo) {
580
- assert (FullStackId != 0 );
581
- FullStackIdToAllocMatchInfo[std::make_pair (
582
- FullStackId, InlinedCallStack.size ())] = {
583
- AllocInfo->Info .getTotalSize (), AllocType};
584
- }
585
- }
586
- }
587
- // If the threshold for the percent of cold bytes is less than 100%,
588
- // and not all bytes are cold, see if we should still hint this
589
- // allocation as cold without context sensitivity.
590
- if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 &&
591
- TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) {
592
- AllocTrie.addSingleAllocTypeAttribute (CI, AllocationType::Cold,
593
- " dominant" );
594
- continue ;
595
- }
596
-
597
- // We might not have matched any to the full inlined call stack.
598
- // But if we did, create and attach metadata, or a function attribute if
599
- // all contexts have identical profiled behavior.
600
- if (!AllocTrie.empty ()) {
601
- NumOfMemProfMatchedAllocs++;
602
- // MemprofMDAttached will be false if a function attribute was
603
- // attached.
604
- bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata (CI);
605
- assert (MemprofMDAttached == I.hasMetadata (LLVMContext::MD_memprof));
606
- if (MemprofMDAttached) {
607
- // Add callsite metadata for the instruction's location list so that
608
- // it simpler later on to identify which part of the MIB contexts
609
- // are from this particular instruction (including during inlining,
610
- // when the callsite metadata will be updated appropriately).
611
- // FIXME: can this be changed to strip out the matching stack
612
- // context ids from the MIB contexts and not add any callsite
613
- // metadata here to save space?
614
- addCallsiteMetadata (I, InlinedCallStack, Ctx);
615
- }
616
- }
625
+ handleAllocSite (I, CI, InlinedCallStack, Ctx, ORE, MaxColdSize,
626
+ AllocInfoIter->second , FullStackIdToAllocMatchInfo);
617
627
continue ;
618
628
}
619
629
0 commit comments