TokenList: avoid Path::identify() being called with empty filename from determineCppC() [skip ci]

Author: firewave

Makefile

@@ -862,7 +862,7 @@ test/testtoken.o: test/testtoken.cpp lib/addoninfo.h lib/check.h lib/color.h lib
 test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenize.cpp
 
-test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
+test/testtokenlist.o: test/testtokenlist.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenlist.cpp
 
 test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h

externals/simplecpp/simplecpp.cpp

@@ -3268,6 +3268,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
     sizeOfType.insert(std::make_pair("double *", sizeof(double *)));
     sizeOfType.insert(std::make_pair("long double *", sizeof(long double *)));
 
+    // use a dummy vector for the macros because as this is not part of the file and would add an empty entry - e.g. /usr/include/poll.h
+    std::vector<std::string> dummy;
+
     const bool hasInclude = (dui.std.size() == 5 && dui.std.compare(0,3,"c++") == 0 && dui.std >= "c++17");
     MacroMap macros;
     for (std::list<std::string>::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) {
@@ -3279,26 +3282,26 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
             continue;
         const std::string lhs(macrostr.substr(0,eq));
         const std::string rhs(eq==std::string::npos ? std::string("1") : macrostr.substr(eq+1));
-        const Macro macro(lhs, rhs, files);
+        const Macro macro(lhs, rhs, dummy);
         macros.insert(std::pair<TokenString,Macro>(macro.name(), macro));
     }
 
-    macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files)));
-    macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", files)));
-    macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files)));
+    macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", dummy)));
+    macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", dummy)));
+    macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", dummy)));
     struct tm ltime = {};
     getLocaltime(ltime);
-    macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(&ltime), files)));
-    macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(&ltime), files)));
+    macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(&ltime), dummy)));
+    macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(&ltime), dummy)));
 
     if (!dui.std.empty()) {
         std::string std_def = simplecpp::getCStdString(dui.std);
         if (!std_def.empty()) {
-            macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, files)));
+            macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy)));
         } else {
             std_def = simplecpp::getCppStdString(dui.std);
             if (!std_def.empty())
-                macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, files)));
+                macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy)));
         }
     }
 

externals/simplecpp/simplecpp.h

@@ -272,6 +272,10 @@ namespace simplecpp {
         /** sizeof(T) */
         std::map<std::string, std::size_t> sizeOfType;
 
+        const std::vector<std::string>& getFiles() const {
+            return files;
+        }
+
     private:
         void combineOperators();
 

lib/tokenlist.cpp

@@ -95,6 +95,7 @@ void TokenList::determineCppC()
 {
     // only try to determine if it wasn't enforced
     if (mLang == Standards::Language::None) {
+        ASSERT_LANG(!getSourceFilePath().empty());
         mLang = Path::identify(getSourceFilePath());
         // TODO: cannot enable assert as this might occur for unknown extensions
         //ASSERT_LANG(mLang != Standards::Language::None);
@@ -372,11 +373,11 @@ bool TokenList::createTokensInternal(std::istream &code, const std::string& file
 // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved)
 void TokenList::createTokens(simplecpp::TokenList&& tokenList)
 {
-    if (tokenList.cfront()) {
+    // tokenList.cfront() might be NULL if the file contained nothing to tokenize so we need to check the files instead
+    if (!tokenList.getFiles().empty()) {
         // this is a copy
-        // TODO: the same as TokenList.files - move that instead
         // TODO: this points to mFiles when called from createTokens(std::istream &, const std::string&)
-        mOrigFiles = mFiles = tokenList.cfront()->location.files;
+        mOrigFiles = mFiles = tokenList.getFiles();
     }
     else
         mFiles.clear();

test/testtokenlist.cpp

@@ -20,13 +20,16 @@
 #include "fixture.h"
 #include "helpers.h"
 #include "platform.h"
+#include "preprocessor.h"
 #include "standards.h"
 #include "token.h"
 #include "tokenlist.h"
 
 #include <sstream>
 #include <string>
 
+#include <simplecpp.h>
+
 class TestTokenList : public TestFixture {
 public:
     TestTokenList() : TestFixture("TestTokenList") {}
@@ -39,6 +42,7 @@ class TestTokenList : public TestFixture {
         TEST_CASE(testaddtoken2);
         TEST_CASE(inc);
         TEST_CASE(isKeyword);
+        TEST_CASE(notokens);
     }
 
     // inspired by #5895
@@ -144,6 +148,19 @@ class TestTokenList : public TestFixture {
             ASSERT_EQUALS(false, tokenlist.front()->isKeyword());
         }
     }
+
+    void notokens() {
+        // analyzing /usr/include/poll.h caused Path::identify() to be called with an empty filename from
+        // TokenList::determineCppC() because there are no tokens
+        const char code[] = "#include <sys/poll.h>";
+        std::istringstream istr(code);
+        std::vector<std::string> files;
+        simplecpp::TokenList tokens1(istr, files, "poll.h", nullptr);
+        Preprocessor preprocessor(settingsDefault, *this);
+        simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, "", files, true);
+        TokenList tokenlist(&settingsDefault);
+        tokenlist.createTokens(std::move(tokensP)); // do not assert
+    }
 };
 
 REGISTER_TEST(TestTokenList)