Fix #14113 (Slow analysis: microchip xc16 code)

externals/simplecpp/simplecpp.cpp

@@ -3072,6 +3072,9 @@ std::pair<simplecpp::FileData *, bool> simplecpp::FileDataCache::tryload(FileDat
     mIdMap.emplace(fileId, data);
     mData.emplace_back(data);
 
+    if (mCallback)
+        mCallback(*data);
+
     return {data, true};
 }
 

externals/simplecpp/simplecpp.h

@@ -9,6 +9,7 @@
 #include <cctype>
 #include <cstdint>
 #include <cstring>
+#include <functional>
 #include <iosfwd>
 #include <list>
 #include <map>
@@ -465,6 +466,12 @@ namespace simplecpp {
             return mData.cend();
         }
 
+        using callback_type = std::function<void (FileData &)>;
+
+        void set_callback(callback_type cb) {
+            mCallback = cb;
+        }
+
     private:
         struct FileID {
 #ifdef _WIN32
@@ -511,6 +518,7 @@ namespace simplecpp {
         container_type mData;
         name_map_type mNameMap;
         id_map_type mIdMap;
+        callback_type mCallback;
     };
 
     SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str);

lib/cppcheck.cpp

@@ -99,9 +99,9 @@ class CppCheck::CppCheckLogger : public ErrorLogger
         closePlist();
     }
 
-    void setRemarkComments(std::vector<RemarkComment> remarkComments)
+    std::vector<RemarkComment>& remarkComments()
     {
-        mRemarkComments = std::move(remarkComments);
+        return mRemarkComments;
     }
 
     void setLocationMacros(const Token* startTok, const std::vector<std::string>& files)
@@ -123,17 +123,25 @@ class CppCheck::CppCheckLogger : public ErrorLogger
         mErrorList.clear();
     }
 
-    void openPlist(const std::string& filename, const std::vector<std::string>& files)
+    void openPlist(const std::string& filename)
     {
         mPlistFile.open(filename);
-        mPlistFile << ErrorLogger::plistHeader(version(), files);
+        mPlistFile << ErrorLogger::plistHeader(version());
+    }
+
+    void setPlistFilenames(std::vector<std::string> files)
+    {
+        if (mPlistFile.is_open()) {
+            mPlistFilenames = std::move(files);
+        }
     }
 
     void closePlist()
     {
         if (mPlistFile.is_open()) {
-            mPlistFile << ErrorLogger::plistFooter();
+            mPlistFile << ErrorLogger::plistFooter(mPlistFilenames);
             mPlistFile.close();
+            mPlistFilenames.clear();
         }
     }
 
@@ -276,6 +284,7 @@ class CppCheck::CppCheckLogger : public ErrorLogger
     std::map<Location, std::set<std::string>> mLocationMacros; // What macros are used on a location?
 
     std::ofstream mPlistFile;
+    std::vector<std::string> mPlistFilenames;
 
     unsigned int mExitCode{};
 
@@ -986,9 +995,6 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
 
         Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
 
-        if (!preprocessor.loadFiles(tokens1, files))
-            return mLogger->exitcode();
-
         if (!mSettings.plistOutput.empty()) {
             std::string filename2;
             if (file.spath().find('/') != std::string::npos)
@@ -997,23 +1003,18 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
                 filename2 = file.spath();
             const std::size_t fileNameHash = std::hash<std::string> {}(file.spath());
             filename2 = mSettings.plistOutput + filename2.substr(0, filename2.find('.')) + "_" + std::to_string(fileNameHash) + ".plist";
-            mLogger->openPlist(filename2, files);
+            mLogger->openPlist(filename2);
         }
 
-        std::string dumpProlog;
+        std::string dumpFooter;
         if (mSettings.dump || !mSettings.addons.empty()) {
-            dumpProlog += getDumpFileContentsRawTokens(files, tokens1);
+            dumpFooter += getDumpFileContentsRawTokensFooter(tokens1);
         }
 
         // Parse comments and then remove them
-        mLogger->setRemarkComments(preprocessor.getRemarkComments(tokens1));
+        preprocessor.addRemarkComments(tokens1, mLogger->remarkComments());
         preprocessor.inlineSuppressions(tokens1, mSuppressions.nomsg);
-        if (mSettings.dump || !mSettings.addons.empty()) {
-            std::ostringstream oss;
-            mSuppressions.nomsg.dump(oss);
-            dumpProlog += oss.str();
-        }
-        preprocessor.removeComments(tokens1);
+        Preprocessor::removeComments(tokens1);
 
         if (!mSettings.buildDir.empty()) {
             analyzerInformation.reset(new AnalyzerInformation);
@@ -1035,19 +1036,39 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         }
 
         // Get directives
-        std::list<Directive> directives = preprocessor.createDirectives(tokens1);
-        preprocessor.simplifyPragmaAsm(tokens1);
+        std::list<Directive> directives;
+        Preprocessor::createDirectives(tokens1, directives);
+        Preprocessor::simplifyPragmaAsm(tokens1);
+
+        // This needs to be a linked list to allow new configurations to be discovered
+        // and added while iterating and checking existing configurations
+        std::list<std::string> configurations;
+        std::set<std::string> configDefines = { "__cplusplus" };
+
+        preprocessor.setLoadCallback([&](simplecpp::FileData& data) {
+            // Do preprocessing on included file
+            preprocessor.addRemarkComments(data.tokens, mLogger->remarkComments());
+            preprocessor.inlineSuppressions(data.tokens, mSuppressions.nomsg);
+            Preprocessor::removeComments(data.tokens);
+            Preprocessor::createDirectives(data.tokens, directives);
+            Preprocessor::simplifyPragmaAsm(data.tokens);
+            // Discover new configurations from included file
+            if ((mSettings.checkAllConfigurations && mSettings.userDefines.empty()) || mSettings.force)
+                preprocessor.getConfigs(data.filename, data.tokens, configDefines, configurations);
+        });
 
         Preprocessor::setPlatformInfo(tokens1, mSettings);
 
         // Get configurations..
-        std::set<std::string> configurations;
         if ((mSettings.checkAllConfigurations && mSettings.userDefines.empty()) || mSettings.force) {
             Timer::run("Preprocessor::getConfigs", mSettings.showtime, &s_timerResults, [&]() {
-                configurations = preprocessor.getConfigs(tokens1);
+                configurations = { "" };
+                preprocessor.getConfigs(file.spath(), tokens1, configDefines, configurations);
+                preprocessor.loadFiles(tokens1, files);
+                configurations.sort();
             });
         } else {
-            configurations.insert(mSettings.userDefines);
+            configurations = { mSettings.userDefines };
         }
 
         if (mSettings.checkConfiguration) {
@@ -1089,7 +1110,6 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         createDumpFile(mSettings, file, fileIndex, fdump, dumpFile);
         if (fdump.is_open()) {
             fdump << getLibraryDumpData();
-            fdump << dumpProlog;
             if (!mSettings.dump)
                 filesDeleter.addFile(dumpFile);
         }
@@ -1272,12 +1292,20 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         }
 
         // TODO: will not be closed if we encountered an exception
-        // dumped all configs, close root </dumps> element now
         if (fdump.is_open()) {
+            // dump all filenames, raw tokens, suppressions
+            std::string dumpHeader = getDumpFileContentsRawTokensHeader(files);
+            fdump << getDumpFileContentsRawTokens(dumpHeader, dumpFooter);
+            mSuppressions.nomsg.dump(fdump);
+            // dumped all configs, close root </dumps> element now
             fdump << "</dumps>" << std::endl;
             fdump.close();
         }
 
+        if (!mSettings.plistOutput.empty()) {
+            mLogger->setPlistFilenames(std::move(files));
+        }
+
         executeAddons(dumpFile, file);
     } catch (const TerminateException &) {
         // Analysis is terminated
@@ -1949,17 +1977,40 @@ bool CppCheck::isPremiumCodingStandardId(const std::string& id) const {
     return false;
 }
 
-std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const {
+std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const
+{
+    std::string header = getDumpFileContentsRawTokensHeader(files);
+    std::string footer = getDumpFileContentsRawTokensFooter(tokens1);
+    return getDumpFileContentsRawTokens(header, footer);
+}
+
+std::string CppCheck::getDumpFileContentsRawTokens(const std::string& header, const std::string& footer)
+{
     std::string dumpProlog;
     dumpProlog += "  <rawtokens>\n";
+    dumpProlog += header;
+    dumpProlog += footer;
+    dumpProlog += "  </rawtokens>\n";
+    return dumpProlog;
+}
+
+std::string CppCheck::getDumpFileContentsRawTokensHeader(const std::vector<std::string>& files) const
+{
+    std::string dumpProlog;
     for (unsigned int i = 0; i < files.size(); ++i) {
         dumpProlog += "    <file index=\"";
         dumpProlog += std::to_string(i);
         dumpProlog += "\" name=\"";
         dumpProlog += ErrorLogger::toxml(Path::getRelativePath(files[i], mSettings.basePaths));
         dumpProlog += "\"/>\n";
     }
-    for (const simplecpp::Token *tok = tokens1.cfront(); tok; tok = tok->next) {
+    return dumpProlog;
+}
+
+std::string CppCheck::getDumpFileContentsRawTokensFooter(const simplecpp::TokenList& tokens1)
+{
+    std::string dumpProlog;
+    for (const simplecpp::Token* tok = tokens1.cfront(); tok; tok = tok->next) {
         dumpProlog += "    <tok ";
 
         dumpProlog += "fileIndex=\"";
@@ -1970,7 +2021,7 @@ std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string
         dumpProlog += std::to_string(tok->location.line);
         dumpProlog += "\" ";
 
-        dumpProlog +="column=\"";
+        dumpProlog += "column=\"";
         dumpProlog += std::to_string(tok->location.col);
         dumpProlog += "\" ";
 
@@ -1980,6 +2031,5 @@ std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string
 
         dumpProlog += "/>\n";
     }
-    dumpProlog += "  </rawtokens>\n";
     return dumpProlog;
 }

lib/cppcheck.h

@@ -154,6 +154,9 @@ class CPPCHECKLIB CppCheck {
      * @brief Get dumpfile <rawtokens> contents, this is only public for testing purposes
      */
     std::string getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const;
+    static std::string getDumpFileContentsRawTokens(const std::string& header, const std::string& footer);
+    std::string getDumpFileContentsRawTokensHeader(const std::vector<std::string>& files) const;
+    static std::string getDumpFileContentsRawTokensFooter(const simplecpp::TokenList& tokens1);
 
     std::string getLibraryDumpData() const;
 

lib/cppcheck.vcxproj

@@ -200,25 +200,25 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <CharacterSet>Unicode</CharacterSet>
     <WholeProgramOptimization>false</WholeProgramOptimization>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>v143</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-PCRE|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <CharacterSet>Unicode</CharacterSet>
     <WholeProgramOptimization>false</WholeProgramOptimization>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>v143</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <CharacterSet>Unicode</CharacterSet>
     <WholeProgramOptimization>false</WholeProgramOptimization>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>v143</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-PCRE|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <CharacterSet>Unicode</CharacterSet>
     <WholeProgramOptimization>false</WholeProgramOptimization>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>v143</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings" />

lib/errorlogger.cpp

@@ -818,20 +818,15 @@ std::string ErrorLogger::toxml(const std::string &str)
     return xml;
 }
 
-std::string ErrorLogger::plistHeader(const std::string &version, const std::vector<std::string> &files)
+std::string ErrorLogger::plistHeader(const std::string &version)
 {
     std::ostringstream ostr;
     ostr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
          << "<!DOCTYPE plist PUBLIC \"-//Apple Computer//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\r\n"
          << "<plist version=\"1.0\">\r\n"
          << "<dict>\r\n"
          << " <key>clang_version</key>\r\n"
-         << "<string>cppcheck version " << version << "</string>\r\n"
-         << " <key>files</key>\r\n"
-         << " <array>\r\n";
-    for (const std::string & file : files)
-        ostr << "  <string>" << ErrorLogger::toxml(file) << "</string>\r\n";
-    ostr << " </array>\r\n"
+         << " <string>cppcheck version " << version << "</string>\r\n"
          << " <key>diagnostics</key>\r\n"
          << " <array>\r\n";
     return ostr.str();

lib/errorlogger.h

@@ -28,6 +28,7 @@
 #include <cstdint>
 #include <list>
 #include <set>
+#include <sstream>
 #include <string>
 #include <vector>
 #include <map>
@@ -268,12 +269,19 @@ class CPPCHECKLIB ErrorLogger {
      */
     static std::string toxml(const std::string &str);
 
-    static std::string plistHeader(const std::string &version, const std::vector<std::string> &files);
+    static std::string plistHeader(const std::string &version);
     static std::string plistData(const ErrorMessage &msg);
-    static const char *plistFooter() {
-        return " </array>\r\n"
-               "</dict>\r\n"
-               "</plist>";
+    static std::string plistFooter(const std::vector<std::string>& files) {
+        std::ostringstream ostr;
+        ostr << " </array>\r\n"
+             << " <key>files</key>\r\n"
+             << " <array>\r\n";
+        for (const std::string& file : files)
+            ostr << "  <string>" << ErrorLogger::toxml(file) << "</string>\r\n";
+        ostr << " </array>\r\n"
+             << "</dict>\r\n"
+             << "</plist>";
+        return ostr.str();
     }
 
     static bool isCriticalErrorId(const std::string& id) {