TokenList: avoid Path::identify() being called with empty filename from determineCppC() (#6326)

Author: firewave

Makefile

@@ -862,7 +862,7 @@ test/testtoken.o: test/testtoken.cpp lib/addoninfo.h lib/check.h lib/color.h lib
 test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenize.cpp
 
-test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
+test/testtokenlist.o: test/testtokenlist.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenlist.cpp
 
 test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h

lib/tokenlist.cpp

@@ -95,6 +95,7 @@ void TokenList::determineCppC()
 {
     // only try to determine if it wasn't enforced
     if (mLang == Standards::Language::None) {
+        ASSERT_LANG(!getSourceFilePath().empty());
         mLang = Path::identify(getSourceFilePath());
         // TODO: cannot enable assert as this might occur for unknown extensions
         //ASSERT_LANG(mLang != Standards::Language::None);
@@ -372,11 +373,11 @@ bool TokenList::createTokensInternal(std::istream &code, const std::string& file
 // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved)
 void TokenList::createTokens(simplecpp::TokenList&& tokenList)
 {
-    if (tokenList.cfront()) {
+    // tokenList.cfront() might be NULL if the file contained nothing to tokenize so we need to check the files instead
+    if (!tokenList.getFiles().empty()) {
         // this is a copy
-        // TODO: the same as TokenList.files - move that instead
         // TODO: this points to mFiles when called from createTokens(std::istream &, const std::string&)
-        mOrigFiles = mFiles = tokenList.cfront()->location.files;
+        mOrigFiles = mFiles = tokenList.getFiles();
     }
     else
         mFiles.clear();

test/testtokenlist.cpp

@@ -20,13 +20,16 @@
 #include "fixture.h"
 #include "helpers.h"
 #include "platform.h"
+#include "preprocessor.h"
 #include "standards.h"
 #include "token.h"
 #include "tokenlist.h"
 
 #include <sstream>
 #include <string>
 
+#include <simplecpp.h>
+
 class TestTokenList : public TestFixture {
 public:
     TestTokenList() : TestFixture("TestTokenList") {}
@@ -39,6 +42,7 @@ class TestTokenList : public TestFixture {
         TEST_CASE(testaddtoken2);
         TEST_CASE(inc);
         TEST_CASE(isKeyword);
+        TEST_CASE(notokens);
     }
 
     // inspired by #5895
@@ -144,6 +148,19 @@ class TestTokenList : public TestFixture {
             ASSERT_EQUALS(false, tokenlist.front()->isKeyword());
         }
     }
+
+    void notokens() {
+        // analyzing /usr/include/poll.h caused Path::identify() to be called with an empty filename from
+        // TokenList::determineCppC() because there are no tokens
+        const char code[] = "#include <sys/poll.h>";
+        std::istringstream istr(code);
+        std::vector<std::string> files;
+        simplecpp::TokenList tokens1(istr, files, "poll.h", nullptr);
+        Preprocessor preprocessor(settingsDefault, *this);
+        simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, "", files, true);
+        TokenList tokenlist(&settingsDefault);
+        tokenlist.createTokens(std::move(tokensP)); // do not assert
+    }
 };
 
 REGISTER_TEST(TestTokenList)