Show Array Node MetaData (#863)

* add ArrayNodes to types

Signed-off-by: Jan Fiedler <jan@union.ai>

* update WorkflowGraph

Signed-off-by: Jan Fiedler <jan@union.ai>

* update execution types

Signed-off-by: Jan Fiedler <jan@union.ai>

* update createExecutionArray

Signed-off-by: Jan Fiedler <jan@union.ai>

* prevent nullref

Signed-off-by: Carina Ursu <carina@union.ai>

* update codecov-action

Signed-off-by: Carina Ursu <carina@union.ai>

---------

Signed-off-by: Jan Fiedler <jan@union.ai>
Signed-off-by: Carina Ursu <carina@union.ai>
Co-authored-by: Carina Ursu <carina@union.ai>

Author: fiedlerNr9

.github/workflows/checks.yml

@@ -37,10 +37,11 @@ jobs:
         run: yarn install  --immutable
       - name: Run tests and generate coverage
         run: make test_unit_codecov
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: .coverage/coverage-final.json
-          fail_ci_if_error: true
+          fail_ci_if_error: false
 
   lint_project:
     runs-on: ubuntu-latest
@@ -68,13 +69,7 @@ jobs:
   release:
     name: Generate Release
     if: ${{ (github.event_name != 'pull_request') && (needs.extract_branch.outputs.branch == 'master') }}
-    needs:
-      [
-        unit_tests_with_coverage,
-        lint_project,
-        build_docker_image,
-        extract_branch,
-      ]
+    needs: [unit_tests_with_coverage, lint_project, build_docker_image, extract_branch]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

packages/oss-console/src/components/Executions/contextProvider/NodeExecutionDetails/createExecutionArray.tsx

@@ -7,6 +7,7 @@ import { Identifier } from '../../../../models/Common/types';
 import { CompiledTask } from '../../../../models/Task/types';
 import { dNode } from '../../../../models/Graph/types';
 import { isEndNode, isStartNode } from '../../../../models/Node/utils';
+import { getTaskTypeFromCompiledNode } from '../../../WorkflowGraph/utils';
 
 interface NodeExecutionInfo extends NodeExecutionDetails {
   scopedId?: string;
@@ -104,6 +105,16 @@ export const getNodeDetails = (
     };
   }
 
+  if (compiledNode?.arrayNode) {
+    returnVal = {
+      ...returnVal,
+      displayType:
+        returnVal.displayType !== NodeExecutionDisplayType.Unknown
+          ? returnVal.displayType
+          : NodeExecutionDisplayType.ArrayNode,
+    };
+  }
+
   return returnVal;
 };
 
@@ -119,6 +130,13 @@ export const getNodeDetailsFromTask = (node: dNode, task?: CompiledTask): NodeEx
     displayType: taskType ?? NodeExecutionDisplayType.Unknown,
   };
 
+  if (node.value?.arrayNode) {
+    returnVal = {
+      ...returnVal,
+      displayType: NodeExecutionDisplayType.ArrayNode,
+    };
+  }
+
   if (node.value?.workflowNode) {
     const { workflowNode } = node.value;
     const info = workflowNode.launchplanRef ?? workflowNode.subWorkflowRef;
@@ -160,7 +178,7 @@ export const getNodeExecutionDetails = (
   node: dNode,
   tasks: CompiledTask[] = [],
 ): NodeExecutionInfo => {
-  const templateName = node?.value?.taskNode?.referenceId?.name ?? node.name;
-  const task = tasks.find((t) => t.template.id.name === templateName);
+  const taskNode = node?.value?.arrayNode?.node?.taskNode || node?.value?.taskNode;
+  const task = getTaskTypeFromCompiledNode(taskNode!, tasks);
   return getNodeDetailsFromTask(node, task);
 };

packages/oss-console/src/components/Executions/types.ts

@@ -1,9 +1,4 @@
-import {
-  NodeExecution,
-  NodeExecutionClosure,
-  NodeExecutionMetadata,
-  WorkflowNodeMetadata,
-} from '../../models/Execution/types';
+import { NodeExecution, NodeExecutionMetadata } from '../../models/Execution/types';
 import { TaskTemplate } from '../../models/Task/types';
 
 export interface ExecutionPhaseConstants {
@@ -15,6 +10,7 @@ export interface ExecutionPhaseConstants {
 }
 
 export enum NodeExecutionDisplayType {
+  ArrayNode = 'Array Node',
   MapTask = 'Map Task',
   BatchHiveTask = 'Hive Batch Task',
   BranchNode = 'Branch Node',

packages/oss-console/src/components/WorkflowGraph/transformerWorkflowToDag.tsx

@@ -6,7 +6,7 @@ import { DISPLAY_NAME_END, DISPLAY_NAME_START } from '../flytegraph/ReactFlow/ut
 import { createDebugLogger } from '../../common/log';
 import { dTypes, dEdge, dNode } from '../../models/Graph/types';
 import { startNodeId, endNodeId } from '../../models/Node/constants';
-import { CompiledNode, ConnectionSet, TaskNode } from '../../models/Node/types';
+import { ArrayNode, CompiledNode, ConnectionSet, TaskNode } from '../../models/Node/types';
 import { CompiledTask } from '../../models/Task/types';
 import { CompiledWorkflow, CompiledWorkflowClosure } from '../../models/Workflow/types';
 import { isStartOrEndNode } from '../../models/Node/utils';
@@ -81,6 +81,7 @@ const createDNode = ({
     gateNode: compiledNode.gateNode,
     level: parentDNode?.level !== undefined ? parentDNode.level + 1 : 0,
     ...nodeMetadata,
+    ...(compiledNode.arrayNode ? { arrayNode: compiledNode.arrayNode } : {}),
     ...(compiledNode.workflowNode ? { workflowNode: compiledNode.workflowNode } : {}),
     ...(compiledNode.gateNode ? { gateNode: compiledNode.gateNode } : {}),
     ...(compiledNode.branchNode ? { taskNode: compiledNode.taskNode } : {}),
@@ -247,6 +248,21 @@ const parseNode = ({
         compiledWorkflowClosure,
       });
     }
+  } else if (node?.arrayNode) {
+    const arrayNode = (node.arrayNode as ArrayNode).node;
+    const taskNode = arrayNode.taskNode as TaskNode;
+    const taskType: CompiledTask = getTaskTypeFromCompiledNode(
+      taskNode,
+      compiledWorkflowClosure.tasks,
+    ) as CompiledTask;
+    dNode = createDNode({
+      compiledNode: node,
+      parentDNode: root,
+      taskTemplate: taskType,
+      nodeMetadataMap,
+      staticExecutionIdsMap,
+      compiledWorkflowClosure,
+    });
   } else if (node.taskNode) {
     const taskNode = node.taskNode as TaskNode;
     const taskType: CompiledTask = getTaskTypeFromCompiledNode(

packages/oss-console/src/components/WorkflowGraph/utils.ts

@@ -100,6 +100,9 @@ export const getSubWorkflowFromId = (
 };
 
 export const getTaskTypeFromCompiledNode = (taskNode: TaskNode, tasks: CompiledTask[]) => {
+  if (!taskNode?.referenceId) {
+    return undefined;
+  }
   for (let i = 0; i < tasks.length; i++) {
     const compiledTask: CompiledTask = tasks[i];
     const taskTemplate: TaskTemplate = compiledTask.template;
@@ -108,7 +111,7 @@ export const getTaskTypeFromCompiledNode = (taskNode: TaskNode, tasks: CompiledT
       return compiledTask;
     }
   }
-  return null;
+  return undefined;
 };
 
 export const getNodeNameFromDag = (dagData: dNode, nodeId: string) => {

packages/oss-console/src/models/Node/types.ts

@@ -6,6 +6,14 @@ export type WorkflowNode = Core.IWorkflowNode;
 /** A graph node indicating a branching decision. */
 export type BranchNode = Core.IBranchNode;
 
+/** A graph node indicating a Array Node. */
+export interface ArrayNode extends Core.IArrayNode {
+  node: CompiledNode;
+  parallelism?: number;
+  minSuccesses?: number;
+  minSuccessRatio?: number;
+}
+
 /** A graph node indicating a task to be executed. This is the most common
  * node type in a Flyte graph.
  */
@@ -30,6 +38,7 @@ export interface CompiledNode extends Core.INode {
   inputs?: Binding[];
   metadata?: CompiledNodeMetadata;
   outputAliases?: Alias[];
+  arrayNode?: ArrayNode;
   taskNode?: TaskNode;
   upstreamNodeIds?: string[];
   workflowNode?: WorkflowNode;