reading constants from plugin and Dockstore API URL as env var

Author: grallewellyn

airflow/dags/appgen_dag.py

@@ -14,19 +14,19 @@
 from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
 from airflow.utils.trigger_rule import TriggerRule
 from kubernetes.client import models as k8s
-from unity_sps_utils import (  # DEFAULT_LOG_LEVEL,; EC2_TYPES,; POD_LABEL,; build_ec2_type_label,
+from unity_sps_utils import (
+    DEFAULT_LOG_LEVEL,
+    EC2_TYPES,
     NODE_POOL_DEFAULT,
     NODE_POOL_HIGH_WORKLOAD,
+    POD_LABEL,
     POD_NAMESPACE,
+    build_ec2_type_label,
     get_affinity,
 )
 
 from airflow import DAG
 
-POD_LABEL = "appgen_pod" + datetime.now().strftime(
-    "%Y%m%d_%H%M%S_%f"
-)  # unique pod label to assure each job runs on its own pod
-
 CONTAINER_RESOURCES = k8s.V1ResourceRequirements(
     requests={
         "ephemeral-storage": "{{ params.request_storage }} ",
@@ -38,86 +38,6 @@
 DOCKERHUB_TOKEN = "/unity/ads/app_gen/development/dockerhub_api_key"
 DOCKSTORE_TOKEN = "/unity/ads/app_gen/development/dockstore_token"
 
-# >>> This part will be removed once the parameters can be imported from unity_sps_plugins.py
-DEFAULT_LOG_LEVEL = 20
-EC2_TYPES = {
-    "t3.micro": {
-        "desc": "General Purpose",
-        "cpu": 1,
-        "memory": 1,
-    },
-    "t3.small": {
-        "desc": "General Purpose",
-        "cpu": 2,
-        "memory": 2,
-    },
-    "t3.medium": {
-        "desc": "General Purpose",
-        "cpu": 2,
-        "memory": 4,
-    },
-    "t3.large": {
-        "desc": "General Purpose",
-        "cpu": 2,
-        "memory": 8,
-    },
-    "t3.xlarge": {
-        "desc": "General Purpose",
-        "cpu": 4,
-        "memory": 16,
-    },
-    "t3.2xlarge": {
-        "desc": "General Purpose",
-        "cpu": 8,
-        "memory": 32,
-    },
-    "r7i.xlarge": {
-        "desc": "Memory Optimized",
-        "cpu": 4,
-        "memory": 32,
-    },
-    "r7i.2xlarge": {
-        "desc": "Memory Optimized",
-        "cpu": 8,
-        "memory": 64,
-    },
-    "r7i.4xlarge": {
-        "desc": "Memory Optimized",
-        "cpu": 16,
-        "memory": 128,
-    },
-    "r7i.8xlarge": {
-        "desc": "Memory Optimized",
-        "cpu": 32,
-        "memory": 256,
-    },
-    "c6i.xlarge": {
-        "desc": "Compute Optimized",
-        "cpu": 4,
-        "memory": 8,
-    },
-    "c6i.2xlarge": {
-        "desc": "Compute Optimized",
-        "cpu": 8,
-        "memory": 16,
-    },
-    "c6i.4xlarge": {
-        "desc": "Compute Optimized",
-        "cpu": 16,
-        "memory": 32,
-    },
-    "c6i.8xlarge": {
-        "desc": "Compute Optimized",
-        "cpu": 32,
-        "memory": 64,
-    },
-}
-
-
-def build_ec2_type_label(key):
-    return f"{key} ({EC2_TYPES.get(key)['desc']}: {EC2_TYPES.get(key)['cpu']}vCPU, {EC2_TYPES.get(key)['memory']}GiB)"
-
-
 # <<<
 LOG_LEVEL_TYPE = {10: "DEBUG", 20: "INFO"}
 
@@ -177,6 +97,7 @@ def build_ec2_type_label(key):
     ),
     k8s.V1EnvVar(name="DOCKERHUB_TOKEN", value="{{ ti.xcom_pull(task_ids='Setup', key='dockerhub_token') }}"),
     k8s.V1EnvVar(name="DOCKSTORE_TOKEN", value="{{ ti.xcom_pull(task_ids='Setup', key='dockstore_token') }}"),
+    k8s.V1EnvVar(name="DOCKSTORE_API_URL", value="http://awslbdockstorestack-lb-1429770210.us-west-2.elb.amazonaws.com:9998/api"),
     k8s.V1EnvVar(name="GITHUB_REPO", value="{{ params.repository }}"),
 ]