Skip to content

Commit 3ea2d6a

Browse files
drewm-swedrewm-swe
authored
Merge pull request #97 from unity-sds/sbg-e2e-concurrent-executions
Sbg e2e concurrent executions
2 parents 5b1acea + cc0ba5e commit 3ea2d6a

File tree

15 files changed

+709
-250
lines changed

15 files changed

+709
-250
lines changed

‎.github/dependabot.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: 'pip'
4+
directory: '/' # location of package manifests
5+
schedule:
6+
interval: 'daily'
7+
time: '09:00'
8+
timezone: 'America/Los_Angeles'
9+
target-branch: 'main'
10+
labels:
11+
- 'dependencies'

‎.github/workflows/integration_tests.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,22 @@ jobs:
4444
# pytest -vv --gherkin-terminal-reporter \
4545
# unity-test/system/integration \
4646
# --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }}
47+
48+
- name: Check Integration Tests Results
49+
if: always()
50+
run: |
51+
dev_status=${{ steps.mcp_venue_dev_integration_tests.outcome }}
52+
# test_status=${{ steps.mcp_venue_test_integration_tests.outcome }}
53+
echo "Dev Integration Tests: $dev_status"
54+
# echo "Test Integration Tests: $test_status"
55+
56+
if [ "$dev_status" != "success" ]; then
57+
echo "MCP Venue Dev Integration Tests failed."
58+
exit 1
59+
fi
60+
61+
# Uncomment this block if MCP Venue Test Integration tests are re-enabled
62+
# if [ "$test_status" != "success" ]; then
63+
# echo "MCP Venue Test Integration Tests failed."
64+
# exit 1
65+
# fi

‎.github/workflows/smoke_tests.yml

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
run: |
5555
pytest -vv --gherkin-terminal-reporter \
5656
unity-test/system/smoke \
57-
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }}
57+
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }} \
5858
--ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_DEV_OGC_PROCESSES_ENDPOINT }}
5959
6060
- name: MCP Venue Test - Smoke tests
@@ -65,7 +65,7 @@ jobs:
6565
run: |
6666
pytest -vv --gherkin-terminal-reporter \
6767
unity-test/system/smoke \
68-
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }}
68+
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }} \
6969
--ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_TEST_OGC_PROCESSES_ENDPOINT }}
7070
7171
- name: MCP Venue Ops - Smoke tests
@@ -75,9 +75,8 @@ jobs:
7575
continue-on-error: true
7676
run: |
7777
pytest -vv --gherkin-terminal-reporter \
78-
unity-test/system/smoke \
79-
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_OPS_AIRFLOW_ENDPOINT || vars.MCP_VENUE_OPS_AIRFLOW_ENDPOINT }}
80-
--ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_OPS_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_OPS_OGC_PROCESSES_ENDPOINT }}
78+
unity-test/system/smoke/step_defs/test_airflow_api_health.py \
79+
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_OPS_AIRFLOW_ENDPOINT || vars.MCP_VENUE_OPS_AIRFLOW_ENDPOINT }} \
8180
8281
- name: MCP Venue SBG Dev - Smoke tests
8382
id: mcp_sbg_dev_smoke_tests
@@ -87,15 +86,36 @@ jobs:
8786
run: |
8887
pytest -vv --gherkin-terminal-reporter \
8988
unity-test/system/smoke \
90-
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT }}
89+
--airflow-endpoint=${{ github.event.inputs.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT }} \
9190
--ogc-processes-endpoint=${{ github.event.inputs.MCP_VENUE_SBG_DEV_OGC_PROCESSES_ENDPOINT || vars.MCP_VENUE_SBG_DEV_OGC_PROCESSES_ENDPOINT }}
9291
9392
# Final step to check outcomes and potentially fail the job
9493
- name: Check Smoke Tests Results
9594
if: always()
9695
run: |
97-
if [ "${{ steps.mcp_venue_dev_smoke_tests.outcome }}" != "success" ] || [ "${{ steps.mcp_venue_test_smoke_tests.outcome }}" != "success" ] || [ "${{ steps.mcp_sbg_dev_smoke_tests.outcome }}" != "success" ]; then
96+
dev_status=${{ steps.mcp_venue_dev_smoke_tests.outcome }}
97+
test_status=${{ steps.mcp_venue_test_smoke_tests.outcome }}
98+
ops_status=${{ steps.mcp_venue_ops_smoke_tests.outcome }}
99+
sbg_dev_status=${{ steps.mcp_sbg_dev_smoke_tests.outcome }}
100+
echo "Dev Smoke Tests: $dev_status"
101+
echo "Test Smoke Tests: $test_status"
102+
echo "Ops Smoke Tests: $ops_status"
103+
echo "SBG Dev Smoke Tests: $sbg_dev_status"
104+
105+
if [ "$dev_status" != "success" ] || [ "$test_status" != "success" ] || [ "$ops_status" != "success" ] || [ "$sbg_dev_status" != "success" ]; then
98106
echo "One or more smoke tests failed."
107+
if [ "$dev_status" != "success" ]; then
108+
echo "MCP Venue Dev Smoke Tests failed."
109+
fi
110+
if [ "$test_status" != "success" ]; then
111+
echo "MCP Venue Test Smoke Tests failed."
112+
fi
113+
if [ "$ops_status" != "success" ]; then
114+
echo "MCP Venue Ops Smoke Tests failed."
115+
fi
116+
if [ "$sbg_dev_status" != "success" ]; then
117+
echo "MCP Venue SBG Dev Smoke Tests failed."
118+
fi
99119
exit 1
100120
else
101121
echo "All smoke tests passed."

‎.github/workflows/unit_tests.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
name: Unit Tests
22

3-
on: [pull_request]
4-
3+
on:
4+
pull_request:
5+
workflow_dispatch:
56
jobs:
67
unit-tests:
78
runs-on: ubuntu-latest
@@ -22,3 +23,5 @@ jobs:
2223
run: |
2324
pytest -vv --gherkin-terminal-reporter \
2425
unity-test/unit
26+
env:
27+
PYTHONPATH: airflow/plugins

‎airflow/dags/karpenter_test.py

Lines changed: 55 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
from datetime import datetime
22

33
from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
4+
from unity_sps_utils import get_affinity
45

56
from airflow import DAG
67

78
POD_NAMESPACE = "airflow"
9+
POD_LABEL = "karpenter_test_task"
810

911
default_args = {
1012
"owner": "unity-sps",
@@ -15,7 +17,7 @@
1517

1618

1719
dag = DAG(
18-
"kubernetes_tasks_with_affinity",
20+
"karpenter_test",
1921
default_args=default_args,
2022
description="DAG with Kubernetes Pod Operators using affinity configurations.",
2123
schedule=None,
@@ -26,147 +28,76 @@
2628

2729
# Define KubernetesPodOperator tasks with default affinity
2830
compute_task = KubernetesPodOperator(
31+
dag=dag,
32+
task_id="compute_task",
2933
namespace=POD_NAMESPACE,
30-
image="hello-world",
3134
name="compute-task",
32-
task_id="compute_task",
35+
image="busybox",
36+
cmds=["sleep"],
37+
arguments=["60"],
38+
retries=3,
39+
in_cluster=True,
3340
get_logs=True,
34-
dag=dag,
35-
is_delete_operator_pod=True,
36-
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
37-
affinity={
38-
"nodeAffinity": {
39-
"preferredDuringSchedulingIgnoredDuringExecution": [
40-
{
41-
"weight": 1,
42-
"preference": {
43-
"matchExpressions": [
44-
{
45-
"key": "karpenter.sh/capacity-type",
46-
"operator": "In",
47-
"values": ["spot"],
48-
}
49-
]
50-
},
51-
}
52-
],
53-
"requiredDuringSchedulingIgnoredDuringExecution": {
54-
"nodeSelectorTerms": [
55-
{
56-
"matchExpressions": [
57-
{
58-
"key": "karpenter.k8s.aws/instance-family",
59-
"operator": "In",
60-
"values": ["c6i", "c5"],
61-
},
62-
{
63-
"key": "karpenter.k8s.aws/instance-cpu",
64-
"operator": "In",
65-
"values": ["2", "4"],
66-
},
67-
]
68-
}
69-
]
70-
},
71-
}
72-
},
41+
container_logs=True,
7342
startup_timeout_seconds=900,
43+
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
44+
labels={"app": POD_LABEL},
45+
annotations={"karpenter.sh/do-not-disrupt": "true"},
46+
affinity=get_affinity(
47+
capacity_type=["spot"],
48+
instance_family=["c6i", "c5"],
49+
instance_cpu=["2", "4"],
50+
anti_affinity_label=POD_LABEL,
51+
),
7452
)
7553

7654
memory_task = KubernetesPodOperator(
55+
dag=dag,
56+
task_id="memory_task",
7757
namespace=POD_NAMESPACE,
78-
image="hello-world",
7958
name="memory-task",
80-
task_id="memory_task",
59+
image="busybox",
60+
cmds=["sleep"],
61+
arguments=["60"],
62+
retries=3,
63+
in_cluster=True,
8164
get_logs=True,
82-
dag=dag,
83-
is_delete_operator_pod=True,
84-
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
85-
affinity={
86-
"nodeAffinity": {
87-
"preferredDuringSchedulingIgnoredDuringExecution": [
88-
{
89-
"weight": 1,
90-
"preference": {
91-
"matchExpressions": [
92-
{
93-
"key": "karpenter.sh/capacity-type",
94-
"operator": "In",
95-
"values": ["spot"],
96-
}
97-
]
98-
},
99-
}
100-
],
101-
"requiredDuringSchedulingIgnoredDuringExecution": {
102-
"nodeSelectorTerms": [
103-
{
104-
"matchExpressions": [
105-
{
106-
"key": "karpenter.k8s.aws/instance-family",
107-
"operator": "In",
108-
"values": ["r6i", "r5"],
109-
},
110-
{
111-
"key": "karpenter.k8s.aws/instance-cpu",
112-
"operator": "In",
113-
"values": ["2", "4"],
114-
},
115-
]
116-
}
117-
]
118-
},
119-
}
120-
},
65+
container_logs=True,
12166
startup_timeout_seconds=900,
67+
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
68+
labels={"app": POD_LABEL},
69+
annotations={"karpenter.sh/do-not-disrupt": "true"},
70+
affinity=get_affinity(
71+
capacity_type=["spot"],
72+
instance_family=["r6i", "r5"],
73+
instance_cpu=["2", "4"],
74+
anti_affinity_label=POD_LABEL,
75+
),
12276
)
12377

78+
12479
general_task = KubernetesPodOperator(
80+
dag=dag,
81+
task_id="general_task",
12582
namespace=POD_NAMESPACE,
126-
image="hello-world",
12783
name="general-task",
128-
task_id="general_task",
84+
image="busybox",
85+
cmds=["sleep"],
86+
arguments=["60"],
87+
retries=3,
88+
in_cluster=True,
12989
get_logs=True,
130-
dag=dag,
131-
is_delete_operator_pod=True,
132-
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
133-
affinity={
134-
"nodeAffinity": {
135-
"preferredDuringSchedulingIgnoredDuringExecution": [
136-
{
137-
"weight": 1,
138-
"preference": {
139-
"matchExpressions": [
140-
{
141-
"key": "karpenter.sh/capacity-type",
142-
"operator": "In",
143-
"values": ["spot"],
144-
}
145-
]
146-
},
147-
}
148-
],
149-
"requiredDuringSchedulingIgnoredDuringExecution": {
150-
"nodeSelectorTerms": [
151-
{
152-
"matchExpressions": [
153-
{
154-
"key": "karpenter.k8s.aws/instance-family",
155-
"operator": "In",
156-
"values": ["m6i", "m5"],
157-
},
158-
{
159-
"key": "karpenter.k8s.aws/instance-cpu",
160-
"operator": "In",
161-
"values": ["2", "4"],
162-
},
163-
]
164-
}
165-
]
166-
},
167-
}
168-
},
90+
container_logs=True,
16991
startup_timeout_seconds=900,
92+
node_selector={"karpenter.sh/nodepool": "airflow-kubernetes-pod-operator"},
93+
labels={"app": POD_LABEL},
94+
annotations={"karpenter.sh/do-not-disrupt": "true"},
95+
affinity=get_affinity(
96+
capacity_type=["spot"],
97+
instance_family=["m6i", "m5"],
98+
instance_cpu=["2", "4"],
99+
anti_affinity_label=POD_LABEL,
100+
),
170101
)
171102

172103
# Task sequence

0 commit comments

Comments
 (0)