From c623675fcaf572c4ab617bf1e3953295ef7c2feb Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 9 Jul 2025 14:16:52 -0700 Subject: [PATCH 1/5] [SampleFDO][TypeProf]Support vtable type profiling for ext-binary and text format --- llvm/include/llvm/ProfileData/SampleProf.h | 99 +++++++++++++++++-- .../llvm/ProfileData/SampleProfReader.h | 12 +++ .../llvm/ProfileData/SampleProfWriter.h | 14 ++- llvm/lib/ProfileData/SampleProf.cpp | 54 +++++++++- llvm/lib/ProfileData/SampleProfReader.cpp | 98 +++++++++++++++++- llvm/lib/ProfileData/SampleProfWriter.cpp | 80 +++++++++++++-- .../Inputs/profile-symbol-list-ext.expected | 44 +++++++++ .../Inputs/sample-profile-ext.proftext | 18 ++++ .../profile-symbol-list-compress.test | 6 ++ .../llvm-profdata/profile-symbol-list.test | 6 ++ llvm/test/tools/llvm-profdata/roundtrip.test | 6 ++ 11 files changed, 412 insertions(+), 25 deletions(-) create mode 100644 llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected create mode 100644 llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 4e1f1131b710f..501d3b17635ed 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -61,7 +61,9 @@ enum class sampleprof_error { ostream_seek_unsupported, uncompress_failed, zlib_unavailable, - hash_mismatch + hash_mismatch, + illegal_line_offset, + duplicate_vtable_type, }; inline std::error_code make_error_code(sampleprof_error E) { @@ -90,6 +92,8 @@ struct is_error_code_enum : std::true_type {}; namespace llvm { namespace sampleprof { +constexpr char kVTableProfPrefix[] = "vtables "; + enum SampleProfileFormat { SPF_None = 0, SPF_Text = 0x1, @@ -203,6 +207,9 @@ enum class SecProfSummaryFlags : uint32_t { /// SecFlagIsPreInlined means this profile contains ShouldBeInlined /// contexts thus this is CS preinliner computed. SecFlagIsPreInlined = (1 << 4), + + /// SecFlagHasVTableTypeProf means this profile contains vtable type profiles. + SecFlagHasVTableTypeProf = (1 << 5), }; enum class SecFuncMetadataFlags : uint32_t { @@ -286,7 +293,7 @@ struct LineLocation { LLVM_ABI void dump() const; // Serialize the line location to the output stream using ULEB128 encoding. - LLVM_ABI void serialize(raw_ostream &OS); + LLVM_ABI void serialize(raw_ostream &OS) const; bool operator<(const LineLocation &O) const { return std::tie(LineOffset, Discriminator) < @@ -302,7 +309,7 @@ struct LineLocation { } uint64_t getHashCode() const { - return ((uint64_t) Discriminator << 32) | LineOffset; + return ((uint64_t)Discriminator << 32) | LineOffset; } uint32_t LineOffset; @@ -317,16 +324,28 @@ struct LineLocationHash { LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); +/// Key represents the id of a vtable and value represents its count. +/// TODO: Rename class FunctionId to SymbolId in a separate PR. +using TypeCountMap = std::map; + +/// Write \p Map to the output stream. Keys are linearized using \p NameTable +/// and written as ULEB128. Values are written as ULEB128 as well. +std::error_code +serializeTypeMap(const TypeCountMap &Map, + const MapVector &NameTable, + raw_ostream &OS); + /// Representation of a single sample record. /// /// A sample record is represented by a positive integer value, which /// indicates how frequently was the associated line location executed. /// /// Additionally, if the associated location contains a function call, -/// the record will hold a list of all the possible called targets. For -/// direct calls, this will be the exact function being invoked. For -/// indirect calls (function pointers, virtual table dispatch), this -/// will be a list of one or more functions. +/// the record will hold a list of all the possible called targets and the types +/// for virtual table dispatches. For direct calls, this will be the exact +/// function being invoked. For indirect calls (function pointers, virtual table +/// dispatch), this will be a list of one or more functions. For virtual table +/// dispatches, this record will also hold the type of the object. class SampleRecord { public: using CallTarget = std::pair; @@ -745,6 +764,7 @@ using BodySampleMap = std::map; // memory, which is *very* significant for large profiles. using FunctionSamplesMap = std::map; using CallsiteSampleMap = std::map; +using CallsiteTypeMap = std::map; using LocToLocMap = std::unordered_map; @@ -927,6 +947,14 @@ class FunctionSamples { return &Iter->second; } + /// Returns the TypeCountMap for inlined callsites at the given \p Loc. + const TypeCountMap *findCallsiteTypeSamplesAt(const LineLocation &Loc) const { + auto Iter = VirtualCallsiteTypeCounts.find(mapIRLocToProfileLoc(Loc)); + if (Iter == VirtualCallsiteTypeCounts.end()) + return nullptr; + return &Iter->second; + } + /// Returns a pointer to FunctionSamples at the given callsite location /// \p Loc with callee \p CalleeName. If no callsite can be found, relax /// the restriction to return the FunctionSamples at callsite location @@ -988,6 +1016,42 @@ class FunctionSamples { return CallsiteSamples; } + /// Return all the callsite type samples collected in the body of the + /// function. + const CallsiteTypeMap &getCallsiteTypeCounts() const { + return VirtualCallsiteTypeCounts; + } + + /// Returns the type samples for the un-drifted location of \p Loc. + TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) { + return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)]; + } + + /// Scale \p Other sample counts by \p Weight and add the scaled result to the + /// type samples for the undrifted location of \p Loc. + template + sampleprof_error addCallsiteVTableTypeProfAt(const LineLocation &Loc, + const T &Other, + uint64_t Weight = 1) { + static_assert((std::is_same_v || + std::is_same_v) && + std::is_same_v, + "T must be a map with StringRef or FunctionId as key and " + "uint64_t as value"); + TypeCountMap &TypeCounts = getTypeSamplesAt(Loc); + bool Overflowed = false; + + for (const auto [Type, Count] : Other) { + FunctionId TypeId(Type); + bool RowOverflow = false; + TypeCounts[TypeId] = SaturatingMultiplyAdd( + Count, Weight, TypeCounts[TypeId], &RowOverflow); + Overflowed |= RowOverflow; + } + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; + } + /// Return the maximum of sample counts in a function body. When SkipCallSite /// is false, which is the default, the return count includes samples in the /// inlined functions. When SkipCallSite is true, the return count only @@ -1042,6 +1106,10 @@ class FunctionSamples { mergeSampleProfErrors(Result, FSMap[Rec.first].merge(Rec.second, Weight)); } + for (const auto &[Loc, OtherTypeMap] : Other.getCallsiteTypeCounts()) + mergeSampleProfErrors( + Result, addCallsiteVTableTypeProfAt(Loc, OtherTypeMap, Weight)); + return Result; } @@ -1285,6 +1353,23 @@ class FunctionSamples { /// collected in the call to baz() at line offset 8. CallsiteSampleMap CallsiteSamples; + /// Map inlined virtual callsites to the vtable from which they are loaded. + /// + /// Each entry is a mapping from the location to the list of vtables and their + /// sampled counts. For example, given: + /// + /// void foo() { + /// ... + /// 5 inlined_vcall_bar(); + /// ... + /// 5 inlined_vcall_baz(); + /// ... + /// 200 inlined_vcall_qux(); + /// } + /// This map will contain two entries. One with two types for line offset 5 + /// and one with one type for line offset 200. + CallsiteTypeMap VirtualCallsiteTypeCounts; + /// IR to profile location map generated by stale profile matching. /// /// Each entry is a mapping from the location on current build to the matched diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index bfe079fbe536f..2e7e2fa65fede 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -703,6 +703,16 @@ class LLVM_ABI SampleProfileReaderBinary : public SampleProfileReader { /// otherwise same as readStringFromTable, also return its hash value. ErrorOr> readSampleContextFromTable(); + std::error_code readBodySampleVTableProf(const LineLocation &Loc, + FunctionSamples &FProfile); + /// Read all callsites' vtable access counts for \p FProfile. + std::error_code readCallsiteVTableProf(FunctionSamples &FProfile); + + /// Read bytes from the input buffer pointed by `Data` and decode them into + /// \p M. `Data` will be advanced to the end of the read bytes when this + /// function returns. Returns error if any. + std::error_code readVTableTypeCountMap(TypeCountMap &M); + /// Points to the current location in the buffer. const uint8_t *Data = nullptr; @@ -727,6 +737,8 @@ class LLVM_ABI SampleProfileReaderBinary : public SampleProfileReader { /// to the start of MD5SampleContextTable. const uint64_t *MD5SampleContextStart = nullptr; + bool ReadVTableProf = false; + private: std::error_code readSummaryEntry(std::vector &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index e84b2095efd7b..9dbeaf56509b0 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -217,13 +217,20 @@ class LLVM_ABI SampleProfileWriterBinary : public SampleProfileWriter { std::error_code writeBody(const FunctionSamples &S); inline void stablizeNameTable(MapVector &NameTable, std::set &V); - + MapVector NameTable; - + void addName(FunctionId FName); virtual void addContext(const SampleContext &Context); void addNames(const FunctionSamples &S); + /// Write \p CallsiteTypeMap to the output stream \p OS. + std::error_code + writeCallsiteVTableProf(const CallsiteTypeMap &CallsiteTypeMap, + raw_ostream &OS); + + bool WriteVTableProf = false; + private: LLVM_ABI friend ErrorOr> SampleProfileWriter::create(std::unique_ptr &OS, @@ -412,8 +419,7 @@ class LLVM_ABI SampleProfileWriterExtBinaryBase class LLVM_ABI SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { public: - SampleProfileWriterExtBinary(std::unique_ptr &OS) - : SampleProfileWriterExtBinaryBase(OS) {} + SampleProfileWriterExtBinary(std::unique_ptr &OS); private: std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index 0c47df3c55dec..759d96da23cf0 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -47,6 +47,24 @@ bool FunctionSamples::ProfileIsPreInlined = false; bool FunctionSamples::UseMD5 = false; bool FunctionSamples::HasUniqSuffix = true; bool FunctionSamples::ProfileIsFS = false; + +std::error_code +serializeTypeMap(const TypeCountMap &Map, + const MapVector &NameTable, + raw_ostream &OS) { + encodeULEB128(Map.size(), OS); + for (const auto &[TypeName, SampleCount] : Map) { + if (auto NameIndexIter = NameTable.find(TypeName); + NameIndexIter != NameTable.end()) { + encodeULEB128(NameIndexIter->second, OS); + } else { + // If the type is not in the name table, we cannot serialize it. + return sampleprof_error::truncated_name_table; + } + encodeULEB128(SampleCount, OS); + } + return sampleprof_error::success; +} } // namespace sampleprof } // namespace llvm @@ -91,6 +109,10 @@ class SampleProfErrorCategoryType : public std::error_category { return "Zlib is unavailable"; case sampleprof_error::hash_mismatch: return "Function hash mismatch"; + case sampleprof_error::illegal_line_offset: + return "Illegal line offset in sample profile data"; + case sampleprof_error::duplicate_vtable_type: + return "Duplicate vtable type in one map"; } llvm_unreachable("A value of sampleprof_error has no message."); } @@ -124,6 +146,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other, for (const auto &I : Other.getCallTargets()) { mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight)); } + return Result; } @@ -150,7 +173,7 @@ std::error_code SampleRecord::serialize( LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); } #endif -void LineLocation::serialize(raw_ostream &OS) { +void LineLocation::serialize(raw_ostream &OS) const { encodeULEB128(LineOffset, OS); encodeULEB128(Discriminator, OS); } @@ -176,6 +199,17 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, return OS; } +static void printTypeCountMap(raw_ostream &OS, LineLocation Loc, + const TypeCountMap &TypeCountMap) { + if (TypeCountMap.empty()) { + return; + } + OS << Loc << ": vtables: "; + for (const auto &[Type, Count] : TypeCountMap) + OS << Type << ":" << Count << " "; + OS << "\n"; +} + /// Print the samples collected for a function on stream \p OS. void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { if (getFunctionHash()) @@ -190,7 +224,13 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { SampleSorter SortedBodySamples(BodySamples); for (const auto &SI : SortedBodySamples.get()) { OS.indent(Indent + 2); + const auto &Loc = SI->first; OS << SI->first << ": " << SI->second; + if (const TypeCountMap *TypeCountMap = + this->findCallsiteTypeSamplesAt(Loc)) { + OS.indent(Indent + 2); + printTypeCountMap(OS, Loc, *TypeCountMap); + } } OS.indent(Indent); OS << "}\n"; @@ -204,11 +244,17 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { SampleSorter SortedCallsiteSamples( CallsiteSamples); for (const auto &CS : SortedCallsiteSamples.get()) { - for (const auto &FS : CS->second) { + for (const auto &[FuncId, FuncSample] : CS->second) { OS.indent(Indent + 2); - OS << CS->first << ": inlined callee: " << FS.second.getFunction() + OS << CS->first << ": inlined callee: " << FuncSample.getFunction() << ": "; - FS.second.print(OS, Indent + 4); + FuncSample.print(OS, Indent + 4); + } + const LineLocation &Loc = CS->first; + auto TypeSamplesIter = VirtualCallsiteTypeCounts.find(Loc); + if (TypeSamplesIter != VirtualCallsiteTypeCounts.end()) { + OS.indent(Indent + 2); + printTypeCountMap(OS, Loc, TypeSamplesIter->second); } } OS.indent(Indent); diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index 6466618aaa655..dd5ffc108fa9a 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -197,8 +197,30 @@ enum class LineType { CallSiteProfile, BodyProfile, Metadata, + VirtualCallTypeProfile, }; +static bool parseTypeCountMap(StringRef Input, + DenseMap &TypeCountMap) { + for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) { + size_t n1 = Input.find(':', Index); + if (n1 == StringRef::npos) + return false; // No colon found, invalid format. + StringRef TypeName = Input.substr(Index, n1 - Index); + // n2 is the start index of count. + size_t n2 = n1 + 1; + // n3 is the start index after the 'target:count' pair. + size_t n3 = Input.find_first_of(' ', n2); + uint64_t Count; + if (Input.substr(n2, n3 - n2).getAsInteger(10, Count)) + return false; // Invalid count. + TypeCountMap[TypeName] = Count; + Index = (n3 == StringRef::npos) ? StringRef::npos + : Input.find_first_not_of(' ', n3); + } + return true; +} + /// Parse \p Input as line sample. /// /// \param Input input line. @@ -215,6 +237,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap &TargetCountMap, + DenseMap &TypeCountMap, uint64_t &FunctionHash, uint32_t &Attributes, bool &IsFlat) { for (Depth = 0; Input[Depth] == ' '; Depth++) @@ -289,6 +312,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, n4 = AfterColon.find_first_of(' '); n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); + // Break the loop if parsing integer succeeded. if (!WordAfterColon.getAsInteger(10, count)) break; @@ -306,6 +330,10 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, // Change n3 to the next blank space after colon + integer pair. n3 = n4; } + } else if (Rest.starts_with(kVTableProfPrefix)) { + LineTy = LineType::VirtualCallTypeProfile; + return parseTypeCountMap(Rest.substr(strlen(kVTableProfPrefix)), + TypeCountMap); } else { LineTy = LineType::CallSiteProfile; size_t n3 = Rest.find_last_of(':'); @@ -374,14 +402,15 @@ std::error_code SampleProfileReaderText::readImpl() { uint64_t NumSamples; StringRef FName; DenseMap TargetCountMap; + DenseMap TypeCountMap; uint32_t Depth, LineOffset, Discriminator; LineType LineTy; uint64_t FunctionHash = 0; uint32_t Attributes = 0; bool IsFlat = false; if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, - Discriminator, FName, TargetCountMap, FunctionHash, - Attributes, IsFlat)) { + Discriminator, FName, TargetCountMap, TypeCountMap, + FunctionHash, Attributes, IsFlat)) { reportError(LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); @@ -410,6 +439,14 @@ std::error_code SampleProfileReaderText::readImpl() { DepthMetadata = 0; break; } + + case LineType::VirtualCallTypeProfile: { + mergeSampleProfErrors( + Result, InlineStack.back()->addCallsiteVTableTypeProfAt( + LineLocation(LineOffset, Discriminator), TypeCountMap)); + break; + } + case LineType::BodyProfile: { FunctionSamples &FProfile = *InlineStack.back(); for (const auto &name_count : TargetCountMap) { @@ -591,6 +628,59 @@ SampleProfileReaderBinary::readSampleContextFromTable() { return std::make_pair(Context, Hash); } +std::error_code +SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) { + auto NumVTableTypes = readNumber(); + if (std::error_code EC = NumVTableTypes.getError()) + return EC; + + for (uint32_t I = 0; I < *NumVTableTypes; ++I) { + auto VTableType(readStringFromTable()); + if (std::error_code EC = VTableType.getError()) + return EC; + + auto VTableSamples = readNumber(); + if (std::error_code EC = VTableSamples.getError()) + return EC; + + if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second) + return sampleprof_error::duplicate_vtable_type; + } + return sampleprof_error::success; +} + +std::error_code +SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) { + if (!ReadVTableProf) + return sampleprof_error::success; + + // Read the vtable type profile for the callsite. + auto NumCallsites = readNumber(); + if (std::error_code EC = NumCallsites.getError()) + return EC; + + for (uint32_t I = 0; I < *NumCallsites; ++I) { + auto LineOffset = readNumber(); + if (std::error_code EC = LineOffset.getError()) + return EC; + + if (!isOffsetLegal(*LineOffset)) + return sampleprof_error::illegal_line_offset; + + auto Discriminator = readNumber(); + if (std::error_code EC = Discriminator.getError()) + return EC; + + // Here we handle FS discriminators: + const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); + + if (std::error_code EC = readVTableTypeCountMap(FProfile.getTypeSamplesAt( + LineLocation(*LineOffset, DiscriminatorVal)))) + return EC; + } + return sampleprof_error::success; +} + std::error_code SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { auto NumSamples = readNumber(); @@ -671,7 +761,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { return EC; } - return sampleprof_error::success; + return readCallsiteVTableProf(FProfile); } std::error_code @@ -733,6 +823,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) FunctionSamples::ProfileIsFS = ProfileIsFS = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf)) + ReadVTableProf = true; break; case SecNameTable: { bool FixedLengthMD5 = diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index 71d2f522fc295..b36f8ebbbf0cf 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -41,6 +41,11 @@ using namespace llvm; using namespace sampleprof; +// To begin with, make this option off by default. +static cl::opt ExtBinaryWriteVTableTypeProf( + "extbinary-write-vtable-type-prof", cl::init(false), cl::Hidden, + cl::desc("Write vtable type profile in ext-binary sample profile writer")); + namespace llvm { namespace support { namespace endian { @@ -435,6 +440,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined); if (Type == SecProfSummary && FunctionSamples::ProfileIsFS) addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator); + if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf) + addSectionFlag(SecProfSummary, + SecProfSummaryFlags::SecFlagHasVTableTypeProf); uint64_t SectionStart = markSectionStart(Type, LayoutIdx); switch (Type) { @@ -478,6 +486,12 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( return sampleprof_error::success; } +SampleProfileWriterExtBinary::SampleProfileWriterExtBinary( + std::unique_ptr &OS) + : SampleProfileWriterExtBinaryBase(OS) { + WriteVTableProf = ExtBinaryWriteVTableTypeProf; +} + std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( const SampleProfileMap &ProfileMap) { // The const indices passed to writeOneSection below are specifying the @@ -587,14 +601,27 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { OS << " " << J.first << ":" << J.second; OS << "\n"; LineCount++; + + if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); + Map && !Map->empty()) { + OS.indent(Indent + 1); + Loc.print(OS); + OS << ": "; + OS << kVTableProfPrefix; + for (const auto [TypeName, Count] : *Map) { + OS << TypeName << ":" << Count << " "; + } + OS << "\n"; + LineCount++; + } } SampleSorter SortedCallsiteSamples( S.getCallsiteSamples()); Indent += 1; - for (const auto &I : SortedCallsiteSamples.get()) + for (const auto &I : SortedCallsiteSamples.get()) { + LineLocation Loc = I->first; for (const auto &FS : I->second) { - LineLocation Loc = I->first; const FunctionSamples &CalleeSamples = FS.second; OS.indent(Indent); Loc.print(OS); @@ -602,6 +629,21 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { if (std::error_code EC = writeSample(CalleeSamples)) return EC; } + + if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); + Map && !Map->empty()) { + OS.indent(Indent); + Loc.print(OS); + OS << ": "; + OS << kVTableProfPrefix; + for (const auto [TypeId, Count] : *Map) { + OS << TypeId << ":" << Count << " "; + } + OS << "\n"; + LineCount++; + } + } + Indent -= 1; if (FunctionSamples::ProfileIsProbeBased) { @@ -661,6 +703,17 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { addName(CalleeSamples.getFunction()); addNames(CalleeSamples); } + + if (!WriteVTableProf) + return; + // Add all the vtable names to NameTable. + for (const auto &VTableAccessCountMap : + llvm::make_second_range(S.getCallsiteTypeCounts())) { + // Add type name to TypeNameTable. + for (const auto Type : llvm::make_first_range(VTableAccessCountMap)) { + addName(Type); + } + } } void SampleProfileWriterExtBinaryBase::addContext( @@ -799,6 +852,21 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader( return sampleprof_error::success; } +std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf( + const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) { + if (!WriteVTableProf) + return sampleprof_error::success; + + encodeULEB128(CallsiteTypeMap.size(), OS); + for (const auto &[Loc, TypeMap] : CallsiteTypeMap) { + Loc.serialize(OS); + if (std::error_code EC = serializeTypeMap(TypeMap, getNameTable(), OS)) + return EC; + } + + return sampleprof_error::success; +} + std::error_code SampleProfileWriterBinary::writeSummary() { auto &OS = *OutputStream; encodeULEB128(Summary->getTotalCount(), OS); @@ -838,14 +906,12 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { encodeULEB128(NumCallsites, OS); for (const auto &J : S.getCallsiteSamples()) for (const auto &FS : J.second) { - LineLocation Loc = J.first; - const FunctionSamples &CalleeSamples = FS.second; - Loc.serialize(OS); - if (std::error_code EC = writeBody(CalleeSamples)) + J.first.serialize(OS); + if (std::error_code EC = writeBody(FS.second)) return EC; } - return sampleprof_error::success; + return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS); } /// Write samples of a top-level function to a binary file. diff --git a/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected b/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected new file mode 100644 index 0000000000000..f7e7499a2c781 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected @@ -0,0 +1,44 @@ +Function: main: 368038, 0, 7 sampled lines +Samples collected in the function's body { + 4: 1068 + 4.2: 1068 + 5: 2150 + 5.1: 2150 + 6: 4160 + 7: 1068 + 9: 4128, calls: _Z3bari:2942 _Z3fooi:1262 + 9: vtables: _ZTVbar:2942 _ZTVfoo:1260 +} +Samples collected in inlined callsites { + 10: inlined callee: inline1: 2000, 0, 1 sampled lines + Samples collected in the function's body { + 1: 2000 + } + No inlined callsites in this function + 10: inlined callee: inline2: 4000, 0, 1 sampled lines + Samples collected in the function's body { + 1: 4000 + } + No inlined callsites in this function + 10: vtables: _ZTVinline1:2000 _ZTVinline2:4000 +} +Function: _Z3bari: 40602, 2874, 1 sampled lines +Samples collected in the function's body { + 1: 2874 +} +No inlined callsites in this function +Function: _Z3fooi: 15422, 1220, 1 sampled lines +Samples collected in the function's body { + 1: 1220 +} +No inlined callsites in this function +======== Dump profile symbol list ======== +_Z3goov +_Z3sumii +__libc_csu_fini +__libc_csu_init +_dl_relocate_static_pie +_fini +_init +_start +main diff --git a/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext b/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext new file mode 100644 index 0000000000000..100133fa17ccb --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext @@ -0,0 +1,18 @@ +main:184019:0 + 4: 534 + 4.2: 534 + 5: 1075 + 5.1: 1075 + 6: 2080 + 7: 534 + 9: 2064 _Z3bari:1471 _Z3fooi:631 + 9: vtables _ZTVbar:1471 _ZTVfoo:630 + 10: inline1:1000 + 1: 1000 + 10: inline2:2000 + 1: 2000 + 10: vtables _ZTVinline1:1000 _ZTVinline2:2000 +_Z3bari:20301:1437 + 1: 1437 +_Z3fooi:7711:610 + 1: 610 diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test b/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test index b445695c8e8e4..06a3ef45ed39a 100644 --- a/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test +++ b/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test @@ -4,3 +4,9 @@ REQUIRES: zlib ; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections %t.1.output %t.2.output -o %t.3.output ; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output ; RUN: diff -b %S/Inputs/profile-symbol-list.expected %t.4.output + +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile-ext.proftext -o %t.1.output +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile-ext.proftext -o %t.2.output +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof %t.1.output %t.2.output -o %t.3.output +; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output +; RUN: diff -b %S/Inputs/profile-symbol-list-ext.expected %t.4.output diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list.test b/llvm/test/tools/llvm-profdata/profile-symbol-list.test index 39dcd11ec1db7..009208001fc61 100644 --- a/llvm/test/tools/llvm-profdata/profile-symbol-list.test +++ b/llvm/test/tools/llvm-profdata/profile-symbol-list.test @@ -7,3 +7,9 @@ ; RUN: llvm-profdata show -sample -show-sec-info-only %t.5.output | FileCheck %s -check-prefix=NOSYMLIST ; NOSYMLIST: ProfileSymbolListSection {{.*}} Size: 0 + +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile-ext.proftext -o %t.1.output +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile-ext.proftext -o %t.2.output +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof %t.1.output %t.2.output -o %t.3.output +; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output +; RUN: diff -b %S/Inputs/profile-symbol-list-ext.expected %t.4.output diff --git a/llvm/test/tools/llvm-profdata/roundtrip.test b/llvm/test/tools/llvm-profdata/roundtrip.test index 7af76e0a58224..eb55534763877 100644 --- a/llvm/test/tools/llvm-profdata/roundtrip.test +++ b/llvm/test/tools/llvm-profdata/roundtrip.test @@ -16,3 +16,9 @@ RUN: llvm-profdata merge --sample --binary -output=%t.4.profdata %S/Inputs/sampl RUN: llvm-profdata merge --sample --extbinary -output=%t.5.profdata %t.4.profdata RUN: llvm-profdata merge --sample --text -output=%t.4.proftext %t.5.profdata RUN: diff -b %t.4.proftext %S/Inputs/sample-profile.proftext +# Round trip from text --> extbinary --> text. +# The vtable profile is supported by ext-binary profile but not raw binary profile format, +# so we don't use raw binary profile format in this roundtrip. +RUN: llvm-profdata merge --sample --extbinary -extbinary-write-vtable-type-prof --output=%t.5.profdata %S/Inputs/sample-profile-ext.proftext +RUN: llvm-profdata merge --sample --text --output=%t.5.proftext %t.5.profdata +RUN: diff -b %t.5.proftext %S/Inputs/sample-profile-ext.proftext From 0b18c2dd4811b7d068e2df25052a64c308c9891c Mon Sep 17 00:00:00 2001 From: mingmingl Date: Thu, 10 Jul 2025 10:22:09 -0700 Subject: [PATCH 2/5] run 'git clang format' --- llvm/include/llvm/ProfileData/SampleProf.h | 2 +- llvm/lib/ProfileData/SampleProf.cpp | 7 +++---- llvm/lib/ProfileData/SampleProfWriter.cpp | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 6b3c2823de9d4..d15b96db58fef 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -309,7 +309,7 @@ struct LineLocation { } uint64_t getHashCode() const { - return ((uint64_t) Discriminator << 32) | LineOffset; + return ((uint64_t)Discriminator << 32) | LineOffset; } uint32_t LineOffset; diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index d9ffec2e0305a..f8706186454af 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -229,7 +229,7 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { if (const TypeCountMap *TypeCountMap = this->findCallsiteTypeSamplesAt(Loc)) { OS.indent(Indent + 2); - printTypeCountMap(OS, Loc, *TypeCountMap); + printTypeCountMap(OS, Loc, *TypeCountMap); } } OS.indent(Indent); @@ -241,7 +241,7 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { OS.indent(Indent); if (!CallsiteSamples.empty()) { OS << "Samples collected in inlined callsites {\n"; - SampleSorter SortedCallsiteSamples( + SampleSorter SortedCallsiteSamples( CallsiteSamples); for (const auto *Element : SortedCallsiteSamples.get()) { // Element is a pointer to a pair of LineLocation and FunctionSamplesMap. @@ -249,8 +249,7 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { for (const FunctionSamples &FuncSample : llvm::make_second_range(FunctionSampleMap)) { OS.indent(Indent + 2); - OS << Loc << ": inlined callee: " << FuncSample.getFunction() - << ": "; + OS << Loc << ": inlined callee: " << FuncSample.getFunction() << ": "; FuncSample.print(OS, Indent + 4); } auto TypeSamplesIter = VirtualCallsiteTypeCounts.find(Loc); diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index a489c1c91ec4f..dfab404a64eb2 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -619,10 +619,11 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { SampleSorter SortedCallsiteSamples( S.getCallsiteSamples()); Indent += 1; - for (const auto* Element : SortedCallsiteSamples.get()) { + for (const auto *Element : SortedCallsiteSamples.get()) { // Element is a pointer to a pair of LineLocation and FunctionSamplesMap. const auto &[Loc, FunctionSamplesMap] = *Element; - for (const FunctionSamples &CalleeSamples : make_second_range(FunctionSamplesMap)) { + for (const FunctionSamples &CalleeSamples : + make_second_range(FunctionSamplesMap)) { OS.indent(Indent); Loc.print(OS); OS << ": "; From 90046d700e1f465b688e905894f71685b33bf3b1 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Thu, 24 Jul 2025 12:06:28 -0700 Subject: [PATCH 3/5] incorporate code review comments --- llvm/include/llvm/ProfileData/SampleProf.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index d15b96db58fef..496d1e940e3f3 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -324,8 +324,10 @@ struct LineLocationHash { LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); -/// Key represents the id of a vtable and value represents its count. -/// TODO: Rename class FunctionId to SymbolId in a separate PR. +/// Key represents type of a C++ polymorphic class type by its vtable and value +/// represents its counter. +/// TODO: The class name FunctionId should be renamed to SymbolId in a refactor +/// change. using TypeCountMap = std::map; /// Write \p Map to the output stream. Keys are linearized using \p NameTable @@ -1016,13 +1018,13 @@ class FunctionSamples { return CallsiteSamples; } - /// Return all the callsite type samples collected in the body of the - /// function. + /// Returns vtable access samples for the C++ types collcted in this function. const CallsiteTypeMap &getCallsiteTypeCounts() const { return VirtualCallsiteTypeCounts; } - /// Returns the type samples for the un-drifted location of \p Loc. + /// Returns the vtable access samples for the C++ types at the un-drifted + /// location of \p Loc. TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) { return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)]; } @@ -1353,10 +1355,10 @@ class FunctionSamples { /// collected in the call to baz() at line offset 8. CallsiteSampleMap CallsiteSamples; - /// Map virtual callsites to the vtable from which they are loaded. + /// Map a virtual callsite to the list of accessed vtables and vtable counts. + /// The callsite is referenced by its source location. /// - /// Each entry is a mapping from the location to the list of vtables and their - /// sampled counts. For example, given: + /// For example, given: /// /// void foo() { /// ... From bfc2b572d84fd6160dc25205bb8ca0ed40d6c35c Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Fri, 25 Jul 2025 11:27:51 -0700 Subject: [PATCH 4/5] Apply suggestion from @paschalis-mpeis Co-authored-by: Paschalis Mpeis --- llvm/include/llvm/ProfileData/SampleProf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 496d1e940e3f3..8e37df7707d7b 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -1018,7 +1018,7 @@ class FunctionSamples { return CallsiteSamples; } - /// Returns vtable access samples for the C++ types collcted in this function. + /// Returns vtable access samples for the C++ types collected in this function. const CallsiteTypeMap &getCallsiteTypeCounts() const { return VirtualCallsiteTypeCounts; } From 93fd3ebf9f850b28aef0d3acae372d41a6bbfbd4 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Fri, 25 Jul 2025 13:15:47 -0700 Subject: [PATCH 5/5] resolve comments --- llvm/include/llvm/ProfileData/SampleProf.h | 10 +++++---- llvm/lib/ProfileData/SampleProf.cpp | 2 -- llvm/lib/ProfileData/SampleProfReader.cpp | 26 ++++++++++++++++------ llvm/lib/ProfileData/SampleProfWriter.cpp | 10 ++++++--- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 8e37df7707d7b..19bef8eda99cf 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -63,7 +63,6 @@ enum class sampleprof_error { zlib_unavailable, hash_mismatch, illegal_line_offset, - duplicate_vtable_type }; inline std::error_code make_error_code(sampleprof_error E) { @@ -1023,14 +1022,17 @@ class FunctionSamples { return VirtualCallsiteTypeCounts; } - /// Returns the vtable access samples for the C++ types at the un-drifted - /// location of \p Loc. + /// Returns the vtable access samples for the C++ types for \p Loc. + /// Under the hood, the caller-specified \p Loc will be un-drifted before the + /// type sample lookup if possible. TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) { return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)]; } /// Scale \p Other sample counts by \p Weight and add the scaled result to the - /// type samples for the undrifted location of \p Loc. + /// type samples for \p Loc. Under the hoold, the caller-provided \p Loc will + /// be un-drifted before the type sample lookup if possible. + /// typename T is either a std::map or a DenseMap. template sampleprof_error addCallsiteVTableTypeProfAt(const LineLocation &Loc, const T &Other, diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index f8706186454af..bf9192c9f2173 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -111,8 +111,6 @@ class SampleProfErrorCategoryType : public std::error_category { return "Function hash mismatch"; case sampleprof_error::illegal_line_offset: return "Illegal line offset in sample profile data"; - case sampleprof_error::duplicate_vtable_type: - return "Duplicate vtable type in one map"; } llvm_unreachable("A value of sampleprof_error has no message."); } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index b8b55f72935b5..f24ca10fc0a7e 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -200,6 +200,8 @@ enum class LineType { VirtualCallTypeProfile, }; +// Parse `Input` as a white-space separated list of `vtable:count` pairs. An +// example input line is `_ZTVbar:1471 _ZTVfoo:630`. static bool parseTypeCountMap(StringRef Input, DenseMap &TypeCountMap) { for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) { @@ -312,7 +314,6 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, n4 = AfterColon.find_first_of(' '); n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); - // Break the loop if parsing integer succeeded. if (!WordAfterColon.getAsInteger(10, count)) break; @@ -642,17 +643,25 @@ SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) { auto VTableSamples = readNumber(); if (std::error_code EC = VTableSamples.getError()) return EC; - - if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second) - return sampleprof_error::duplicate_vtable_type; + // The source profile should not have duplicate vtable records at the same + // location. In case duplicate vtables are found, reader can emit a warning + // but continue processing the profile. + if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second) { + Ctx.diagnose(DiagnosticInfoSampleProfile( + Buffer->getBufferIdentifier(), 0, + "Duplicate vtable type " + VTableType->str() + + " at the same location. Additional counters will be ignored.", + DS_Warning)); + continue; + } } return sampleprof_error::success; } std::error_code SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) { - if (!ReadVTableProf) - return sampleprof_error::success; + assert(ReadVTableProf && + "Cannot read vtable profiles if ReadVTableProf is false"); // Read the vtable type profile for the callsite. auto NumCallsites = readNumber(); @@ -761,7 +770,10 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { return EC; } - return readCallsiteVTableProf(FProfile); + if (ReadVTableProf) + return readCallsiteVTableProf(FProfile); + + return sampleprof_error::success; } std::error_code diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index dfab404a64eb2..e5f31348578b8 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -855,8 +855,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader( std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf( const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) { - if (!WriteVTableProf) - return sampleprof_error::success; + assert(WriteVTableProf && + "writeCallsiteVTableProf should not be called if WriteVTableProf is " + "false"); encodeULEB128(CallsiteTypeMap.size(), OS); for (const auto &[Loc, TypeMap] : CallsiteTypeMap) { @@ -912,7 +913,10 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { return EC; } - return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS); + if (WriteVTableProf) + return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS); + + return sampleprof_error::success; } /// Write samples of a top-level function to a binary file.