implement yaml native bulk transpilation

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -47,6 +47,24 @@ const getThreadsCount = () => {
   return 3; // Default (like the workerpool do)
 };
 
+const splitFilesToChunks = (files: FileContent[], chunksCount: number): FileContent[][] => {
+  let chunks: FileContent[][];
+  if (files.length < chunksCount * chunksCount) {
+    chunks = [files];
+  } else {
+    const baseSize = Math.floor(files.length / chunksCount);
+    chunks = [];
+    for (let i = 0; i < chunksCount; i++) {
+      // For the last part, we take the remaining files so we don't lose the extra ones.
+      const start = i * baseSize;
+      const end = (i === chunksCount - 1) ? files.length : start + baseSize;
+      chunks.push(files.slice(start, end));
+    }
+  }
+
+  return chunks;
+};
+
 export type DataSchemaCompilerOptions = {
   compilerCache: CompilerCache;
   omitErrors?: boolean;
@@ -288,8 +306,31 @@ export class DataSchemaCompiler {
       }
 
       if (transpilationNative) {
-        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js') || file.convertedToJs);
-        let JsFilesTasks = [];
+        const jsFiles: FileContent[] = [];
+        const yamlFiles: FileContent[] = [];
+        const jinjaFiles: FileContent[] = [];
+        const otherFiles: FileContent[] = [];
+
+        toCompile.forEach((file) => {
+          if (file.fileName.endsWith('.js') || file.convertedToJs) {
+            jsFiles.push(file);
+          } else if (file.fileName.endsWith('.jinja') ||
+            (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml'))
+            && file.content.match(JINJA_SYNTAX)
+          ) {
+            jinjaFiles.push(file);
+          } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
+            yamlFiles.push(file);
+          } else {
+            otherFiles.push(file);
+          }
+        });
+
+        let jsFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
+        let yamlFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
+        // XXX: For now we don't have bulk transpilation for jinja files
+        // let jinjaFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
+        // let otherFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
 
         if (jsFiles.length > 0) {
           // Warming up swc compiler cache
@@ -300,26 +341,19 @@ export class DataSchemaCompiler {
 
           await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
 
-          let jsChunks;
-          if (jsFiles.length < transpilationNativeThreadsCount * transpilationNativeThreadsCount) {
-            jsChunks = [jsFiles];
-          } else {
-            const baseSize = Math.floor(jsFiles.length / transpilationNativeThreadsCount);
-            jsChunks = [];
-            for (let i = 0; i < transpilationNativeThreadsCount; i++) {
-              // For the last part, we take the remaining files so we don't lose the extra ones.
-              const start = i * baseSize;
-              const end = (i === transpilationNativeThreadsCount - 1) ? jsFiles.length : start + baseSize;
-              jsChunks.push(jsFiles.slice(start, end));
-            }
-          }
-          JsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesBulk(chunk, errorsReport, { transpilerNames, compilerId }));
+          const jsChunks = splitFilesToChunks(jsFiles, transpilationNativeThreadsCount);
+          jsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesNativeBulk(chunk, errorsReport, { transpilerNames, compilerId }));
         }
 
-        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js') && !file.convertedToJs)
-          .map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames, compilerId }));
+        if (yamlFiles.length > 0) {
+          const yamlChunks = splitFilesToChunks(yamlFiles, transpilationNativeThreadsCount);
+          yamlFilesTasks = yamlChunks.map(chunk => this.transpileYamlFilesNativeBulk(chunk, errorsReport, { transpilerNames, compilerId }));
+        }
 
-        results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat();
+        const jinjaAndOtherFilesTasks = jinjaFiles.concat(otherFiles)
+          .map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }));
+
+        results = (await Promise.all([...jsFilesTasks, ...yamlFilesTasks, ...jinjaAndOtherFilesTasks])).flat();
       } else if (transpilationWorkerThreads) {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
@@ -521,11 +555,7 @@ export class DataSchemaCompiler {
     }
   }
 
-  /**
-   * Right now it is used only for transpilation in native,
-   * so no checks for transpilation type inside this method
-   */
-  private async transpileJsFilesBulk(
+  private async transpileJsFilesNativeBulk(
     files: FileContent[],
     errorsReport: ErrorReporter,
     { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions
@@ -563,6 +593,36 @@ export class DataSchemaCompiler {
     });
   }
 
+  private async transpileYamlFilesNativeBulk(
+    files: FileContent[],
+    errorsReport: ErrorReporter,
+    { compilerId }: TranspileOptions
+  ): Promise<(FileContent | undefined)[]> {
+    const reqDataArr = files.map(file => ({
+      fileName: file.fileName,
+      fileContent: file.content,
+      transpilers: [],
+      compilerId: compilerId || '',
+    }));
+    const res = await transpileYaml(reqDataArr);
+
+    return files.map((file, index) => {
+      errorsReport.inFile(file);
+      if (!res[index]) { // This should not happen in theory but just to be safe
+        errorsReport.error(`No transpilation result received for the file ${file.fileName}.`);
+        return undefined;
+      }
+      errorsReport.addErrors(res[index].errors);
+      errorsReport.addWarnings(res[index].warnings as unknown as SyntaxErrorInterface[]);
+      errorsReport.exitFile();
+
+      file.content = res[index].code;
+      file.convertedToJs = true;
+
+      return { ...file };
+    });
+  }
+
   private async transpileJsFile(
     file: FileContent,
     errorsReport: ErrorReporter,