Fix stubs (#648)

* Fix stubs value generation
* Disable clion integration tests

Author: ladisgin

.github/workflows/build-utbot.yml

@@ -94,23 +94,23 @@ jobs:
           ./docker/action-scripts/build-vsix.sh
           chmod +x docker/action-scripts/integration-tests.sh
           ./docker/action-scripts/integration-tests.sh
-      - name: Setup Java for building CLion plugin
-        uses: actions/setup-java@v3
-        with:
-          distribution: zulu
-          java-version: 11
-      - name: Run CLion integration tests
-        run: |
-          chmod +x docker/action-scripts/runClionIntegrationTests.sh 
-          ./docker/action-scripts/runClionIntegrationTests.sh
-      - name: Upload logs
-        uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: test-report
-          path: |
-            /github/home/logs
-            clion-plugin/build/reports/tests/**/*
+#      - name: Setup Java for building CLion plugin
+#        uses: actions/setup-java@v3
+#        with:
+#          distribution: zulu
+#          java-version: 11
+#      - name: Run CLion integration tests
+#        run: |
+#          chmod +x docker/action-scripts/runClionIntegrationTests.sh
+#          ./docker/action-scripts/runClionIntegrationTests.sh
+#      - name: Upload logs
+#        uses: actions/upload-artifact@v3
+#        if: failure()
+#        with:
+#          name: test-report
+#          path: |
+#            /github/home/logs
+#            clion-plugin/build/reports/tests/**/*
 
   build-utbot-and-generate-test:
     needs: matrix-prep

server/src/ReturnTypesFetcher.cpp

@@ -3,25 +3,32 @@
 #include "fetchers/Fetcher.h"
 #include "testgens/BaseTestGen.h"
 
-void ReturnTypesFetcher::fetch(ProgressWriter *const progressWriter,
-                               const CollectionUtils::FileSet &allFiles) {
-    tests::TestsMap testsMap;
-    for (const auto &filePath : allFiles) {
-        testsMap[filePath];
-    }
-    Fetcher(Fetcher::Options::Value::RETURN_TYPE_NAMES_ONLY,
-                 testGen->getTargetBuildDatabase()->compilationDatabase, testsMap, nullptr, nullptr,
-                 testGen->compileCommandsJsonPath, false)
-        .fetchWithProgress(progressWriter, "Fetching return types for functions", true);
-    for (auto const &[sourceFilePath, test] : testsMap) {
-        for (const auto& [methodName, methodDescription]: test.methods) {
-            auto returnTypedefName = PrinterUtils::getReturnMangledName(methodDescription.name);
-            if (CollectionUtils::containsKey(methodDescription.functionPointers, returnTypedefName)) {
-                types::Type returnTypeCopy = methodDescription.returnType;
-                returnTypeCopy.replaceUsedType(returnTypedefName);
-                testGen->methodNameToReturnTypeMap[methodName] = returnTypeCopy;
-            } else {
-                testGen->methodNameToReturnTypeMap[methodName] = methodDescription.returnType;
+void ReturnTypesFetcher::fetch(const ProgressWriter *progressWriter) {
+    const std::shared_ptr<CompilationDatabase> buildDatabases[] = {
+            testGen->getTargetBuildDatabase()->compilationDatabase,
+            testGen->getProjectBuildDatabase()->compilationDatabase};
+    for (const auto &buildDatabase: buildDatabases) {
+        tests::TestsMap testsMap;
+        for (const auto &filePath: buildDatabase->getAllFiles()) {
+            testsMap[filePath];
+        }
+        Fetcher(Fetcher::Options::Value::RETURN_TYPE_NAMES_ONLY,
+                buildDatabase, testsMap, nullptr, nullptr,
+                testGen->compileCommandsJsonPath, false)
+                .fetchWithProgress(progressWriter, "Fetching return types for functions", true);
+        for (auto const &[sourceFilePath, test]: testsMap) {
+            for (const auto &[methodName, methodDescription]: test.methods) {
+                if (testGen->methodNameToReturnTypeMap.count(methodName)) {
+                    continue;
+                }
+                auto returnTypedefName = PrinterUtils::getReturnMangledName(methodDescription.name);
+                if (CollectionUtils::containsKey(methodDescription.functionPointers, returnTypedefName)) {
+                    types::Type returnTypeCopy = methodDescription.returnType;
+                    returnTypeCopy.replaceUsedType(returnTypedefName);
+                    testGen->methodNameToReturnTypeMap[methodName] = returnTypeCopy;
+                } else {
+                    testGen->methodNameToReturnTypeMap[methodName] = methodDescription.returnType;
+                }
             }
         }
     }

server/src/ReturnTypesFetcher.h

@@ -18,7 +18,7 @@ class ReturnTypesFetcher {
     explicit ReturnTypesFetcher(BaseTestGen *testGen) : testGen(testGen) {
     }
 
-    void fetch(ProgressWriter *const progressWriter, const CollectionUtils::FileSet &allFiles);
+    void fetch(const ProgressWriter *progressWriter);
 };
 
 

server/src/Server.cpp

@@ -206,10 +206,10 @@ Status Server::TestsGenServiceImpl::ProcessBaseTestRequest(BaseTestGen &testGen,
         types::TypesHandler typesHandler{testGen.types, sizeContext};
         testGen.progressWriter->writeProgress("Generating stub files", 0.0);
         StubGen stubGen(testGen);
-
-        Synchronizer synchronizer(&testGen, &sizeContext);
-        synchronizer.synchronize(typesHandler);
-
+        {
+            Synchronizer synchronizer(&testGen, &sizeContext);
+            synchronizer.synchronize(typesHandler);
+        }
         std::shared_ptr<LineInfo> lineInfo = nullptr;
         auto lineTestGen = dynamic_cast<LineTestGen *>(&testGen);
 
@@ -253,9 +253,10 @@ Status Server::TestsGenServiceImpl::ProcessBaseTestRequest(BaseTestGen &testGen,
             }
         }
         auto generator = std::make_shared<KleeGenerator>(&testGen, typesHandler, pathSubstitution);
-
-        ReturnTypesFetcher returnTypesFetcher{&testGen};
-        returnTypesFetcher.fetch(testGen.progressWriter, synchronizer.getTargetSourceFiles());
+        {
+            ReturnTypesFetcher returnTypesFetcher(&testGen);
+            returnTypesFetcher.fetch(testGen.progressWriter);
+        }
         LOG_S(DEBUG) << "Temporary build directory path: " << testGen.serverBuildDir;
         generator->buildKleeFiles(testGen.tests, lineInfo);
         generator->handleFailedFunctions(testGen.tests);
@@ -562,8 +563,10 @@ Status Server::TestsGenServiceImpl::ProcessProjectStubsRequest(BaseTestGen *test
                     testGen->compileCommandsJsonPath, false);
 
     fetcher.fetchWithProgress(testGen->progressWriter, logMessage);
-    Synchronizer synchronizer(testGen, &sizeContext);
-    synchronizer.synchronize(typesHandler);
+    {
+        Synchronizer synchronizer(testGen, &sizeContext);
+        synchronizer.synchronize(typesHandler);
+    }
     stubsWriter->writeResponse(testGen->synchronizedStubs, testGen->projectContext.testDirPath);
     return Status::OK;
 }

server/src/Synchronizer.cpp

@@ -88,10 +88,11 @@ bool Synchronizer::isProbablyOutdatedWrappers(const fs::path &srcFilePath) const
 }
 
 CollectionUtils::FileSet Synchronizer::getOutdatedSourcePaths() const {
-    auto allFiles = getTargetSourceFiles();
-    auto outdatedSources = CollectionUtils::filterOut(getTargetSourceFiles(), [this](fs::path const &sourcePath) {
-        return !isProbablyOutdatedWrappers(sourcePath);
-    });
+    auto allFiles = testGen->getTargetSourceFiles();
+    auto outdatedSources = CollectionUtils::filterOut(testGen->getTargetSourceFiles(),
+                                                      [this](fs::path const &sourcePath) {
+                                                          return !isProbablyOutdatedWrappers(sourcePath);
+                                                      });
     return outdatedSources;
 }
 
@@ -199,15 +200,14 @@ void Synchronizer::synchronizeStubs(StubSet &outdatedStubs,
     for (const StubOperator &outdatedStub : outdatedStubs) {
         fs::path stubPath = outdatedStub.getStubPath(testGen->projectContext);
         Tests const &methodDescription = stubFilesMap[stubPath];
+        tests::Tests tests = StubGen::mergeSourceFileIntoStub(
+                methodDescription, sourceFilesMap.at(outdatedStub.getSourceFilePath()));
         if (outdatedStub.isHeader()) {
-            std::string code = sourceToHeaderRewriter.generateStubHeader(outdatedStub.getSourceFilePath());
+            std::string code = sourceToHeaderRewriter.generateStubHeader(tests, outdatedStub.getSourceFilePath());
             testGen->synchronizedStubs.emplace_back(stubPath, code);
         } else {
-            tests::Tests newStubFile = StubGen::mergeSourceFileIntoStub(
-                methodDescription, sourceFilesMap.at(outdatedStub.getSourceFilePath()));
             printer::StubsPrinter stubsPrinter(Paths::getSourceLanguage(stubPath));
-            Stubs stubFile =
-                stubsPrinter.genStubFile(newStubFile, typesHandler, testGen->projectContext);
+            Stubs stubFile = stubsPrinter.genStubFile(tests, typesHandler, testGen->projectContext);
             testGen->synchronizedStubs.emplace_back(stubFile);
         }
     }
@@ -227,7 +227,7 @@ Synchronizer::createStubsCompilationDatabase(StubSet &stubFiles,
 void Synchronizer::synchronizeWrappers(const CollectionUtils::FileSet &outdatedSourcePaths,
                                        const types::TypesHandler &typesHandler) const {
     auto sourceFilesNeedToRegenerateWrappers = outdatedSourcePaths;
-    for (fs::path const &sourceFilePath : getTargetSourceFiles()) {
+    for (fs::path const &sourceFilePath: testGen->getTargetSourceFiles()) {
         if (!CollectionUtils::contains(sourceFilesNeedToRegenerateWrappers, sourceFilePath)) {
             auto wrapperFilePath =
                 Paths::getWrapperFilePath(testGen->projectContext, sourceFilePath);
@@ -247,27 +247,19 @@ void Synchronizer::synchronizeWrappers(const CollectionUtils::FileSet &outdatedS
         });
 }
 
-const CollectionUtils::FileSet &Synchronizer::getTargetSourceFiles() const {
-    return testGen->getTargetBuildDatabase()->compilationDatabase->getAllFiles();
-}
-
-const CollectionUtils::FileSet &Synchronizer::getProjectSourceFiles() const {
-    return testGen->getProjectBuildDatabase()->compilationDatabase->getAllFiles();
-}
-
 StubSet Synchronizer::getStubsFiles() const {
     return getStubSetFromSources(testGen->getProjectBuildDatabase()->compilationDatabase->getAllFiles());
 }
 
 void Synchronizer::prepareDirectory(const fs::path &stubDirectory) {
     fs::create_directories(stubDirectory);
-    for (const auto &entry : fs::recursive_directory_iterator(stubDirectory)) {
+    for (const auto &entry: fs::recursive_directory_iterator(stubDirectory)) {
         if (entry.is_regular_file()) {
             fs::path stubPath = entry.path();
             if (!Paths::isHeaderFile(stubPath)) {
                 fs::path sourcePath =
-                    Paths::stubPathToSourcePath(testGen->projectContext, stubPath);
-                if (!CollectionUtils::contains(getProjectSourceFiles(), sourcePath)) {
+                        Paths::stubPathToSourcePath(testGen->projectContext, stubPath);
+                if (!CollectionUtils::contains(testGen->getProjectSourceFiles(), sourcePath)) {
                     LOG_S(DEBUG) << "Found extra file in stub directory: " << stubPath
                                  << ". Removing it.";
                     fs::remove(stubPath);

server/src/Synchronizer.h

@@ -27,29 +27,33 @@ class Synchronizer {
 
     [[nodiscard]] std::unordered_set<StubOperator, HashUtils::StubHash> getOutdatedStubs() const;
 
-    long long getFileOutdatedTime(const fs::path &filePath) const;
+    [[nodiscard]] long long getFileOutdatedTime(const fs::path &filePath) const;
 
-    bool isProbablyOutdatedStubs(const fs::path &srcFilePath) const;
+    [[nodiscard]] bool isProbablyOutdatedStubs(const fs::path &srcFilePath) const;
 
-    bool isProbablyOutdatedWrappers(const fs::path &srcFilePath) const;
+    [[nodiscard]] bool isProbablyOutdatedWrappers(const fs::path &srcFilePath) const;
 
-    bool removeStubIfSourceAbsent(const StubOperator &stub) const;
+    [[nodiscard]] bool removeStubIfSourceAbsent(const StubOperator &stub) const;
 
     void synchronizeStubs(std::unordered_set<StubOperator, HashUtils::StubHash> &outdatedStubs,
                           const types::TypesHandler &typesHandler);
+
     void synchronizeWrappers(const CollectionUtils::FileSet &outdatedSourcePaths,
                              const types::TypesHandler &typesHandler) const;
 
     std::shared_ptr<CompilationDatabase>
     createStubsCompilationDatabase(
-        std::unordered_set<StubOperator, HashUtils::StubHash> &stubFiles,
-        const fs::path &ccJsonStubDirPath) const;
+            std::unordered_set<StubOperator, HashUtils::StubHash> &stubFiles,
+            const fs::path &ccJsonStubDirPath) const;
 
-    void prepareDirectory(fs::path const& stubDirectory);
+    void prepareDirectory(fs::path const &stubDirectory);
 
     static std::unordered_set<StubOperator, HashUtils::StubHash>
 
     getStubSetFromSources(const CollectionUtils::FileSet &paths);
+
+    [[nodiscard]] std::unordered_set<StubOperator, HashUtils::StubHash> getStubsFiles() const;
+
 public:
     static std::unordered_set<StubOperator, HashUtils::StubHash>
 
@@ -60,10 +64,6 @@ class Synchronizer {
     Synchronizer(BaseTestGen *testGen, types::TypesHandler::SizeContext *sizeContext);
 
     void synchronize(const types::TypesHandler &typesHandler);
-
-    [[nodiscard]] const CollectionUtils::FileSet &getTargetSourceFiles() const;
-    [[nodiscard]] const CollectionUtils::FileSet &getProjectSourceFiles() const;
-    [[nodiscard]] std::unordered_set<StubOperator, HashUtils::StubHash> getStubsFiles() const;
 };