Skip to content

Commit 163c366

Browse files
GMartinez-SistiGMartinez-Sisti
authored
chore(cleanup): move scripts to files (#364)
1 parent 0b66994 commit 163c366

File tree

5 files changed

+154
-111
lines changed

5 files changed

+154
-111
lines changed

‎.github/actions/cleanup-aws/action.yaml‎

Lines changed: 11 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
1-
name: "Cleanup AWS leftovers"
2-
description: "Cleanup any AWS leftovers of e2e test"
1+
name: Cleanup AWS leftovers
2+
description: Cleanup any AWS leftovers of e2e test
33
inputs:
44
service_name:
55
required: true
6-
description: "The full name of the service for filtering purposes"
6+
description: The full name of the service for filtering purposes
77
aws_role_arn:
88
required: true
9-
description: "The ARN of the AWS role to assume for AWS tests"
9+
description: The ARN of the AWS role to assume for AWS tests
1010
aws_region:
1111
required: true
12-
description: "The AWS region to use for AWS tests"
12+
description: The AWS region to use for AWS tests
1313

1414
runs:
15-
using: "composite"
15+
using: composite
1616
steps:
1717
- name: Configure AWS credentials
1818
id: aws-auth
@@ -21,53 +21,10 @@ runs:
2121
role-to-assume: ${{ inputs.aws_role_arn }}
2222
aws-region: ${{ inputs.aws_region }}
2323

24-
- name: Cleanup AWS
24+
- name: Cleanup ClickHouse Instances
2525
shell: bash
26+
env:
27+
AWS_REGION: ${{ inputs.aws_region }}
28+
SERVICE_NAME: ${{ inputs.service_name }}
2629
run: |
27-
echo "::group::Deleting VPC Endpoints"
28-
attempts=10
29-
while [ $attempts -gt 0 ]; do
30-
attempts=$((attempts - 1))
31-
endpointids="$(aws ec2 --region ${{ inputs.aws_region }} describe-vpc-endpoints|jq --arg name "${{ inputs.service_name }}" -r '.VpcEndpoints[] | select(.Tags[0].Value == $name) | .VpcEndpointId')"
32-
if [ "$endpointids" == "" ]
33-
then
34-
break
35-
fi
36-
37-
echo "Deleting endpoints $endpointids"
38-
for endpointid in $endpointids
39-
do
40-
echo "Deleting vpc endpoint $endpointid"
41-
aws ec2 --region "${{ inputs.aws_region }}" delete-vpc-endpoints --vpc-endpoint-ids "$endpointid"
42-
done
43-
44-
sleep 60
45-
done
46-
echo "::endgroup::"
47-
48-
echo "::group::Deleting Security Groups"
49-
sgids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-security-groups|jq --arg name "${{ inputs.service_name }}" -r '.SecurityGroups[] | select(.Tags[0].Value == $name) | .GroupId')"
50-
for sgid in $sgids
51-
do
52-
echo "Deleting SG $sgid"
53-
aws ec2 --region "${{ inputs.aws_region }}" delete-security-group --group-id "$sgid"
54-
done
55-
echo "::endgroup::"
56-
57-
echo "::group::Deleting Subnets"
58-
subnetids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-subnets|jq --arg name "${{ inputs.service_name }}" -r '.Subnets[] | select(.Tags[0].Value == $name) | .SubnetId')"
59-
for subnetid in $subnetids
60-
do
61-
echo "Deleting subnet $subnetid"
62-
aws ec2 --region "${{ inputs.aws_region }}" delete-subnet --subnet-id "$subnetid"
63-
done
64-
echo "::endgroup::"
65-
66-
echo "::group::Deleting VPCs"
67-
vpcids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-vpcs|jq --arg name "${{ inputs.service_name }}" -r '.Vpcs[] | select(.Tags[0].Value == $name) | .VpcId')"
68-
for vpcid in $vpcids
69-
do
70-
echo "Deleting vpc $vpcid"
71-
aws ec2 --region "${{ inputs.aws_region }}" delete-vpc --vpc-id "$vpcid"
72-
done
73-
echo "::endgroup::"
30+
./.github/actions/cleanup-aws/cleanup.sh

‎.github/actions/cleanup-aws/cleanup.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
AWS_REGION="${AWS_REGION:?"AWS_REGION cannot be empty"}"
6+
SERVICE_NAME="${SERVICE_NAME:?"SERVICE_NAME cannot be empty"}"
7+
8+
### VPC Endpoints
9+
echo "::group::Deleting VPC Endpoints..."
10+
ATTEMPTS=10
11+
while [ $ATTEMPTS -gt 0 ]; do
12+
ATTEMPTS=$((ATTEMPTS - 1))
13+
14+
mapfile -t ENDPOINT_IDS < <(aws ec2 --region "${AWS_REGION}" describe-vpc-endpoints | jq --arg name "${SERVICE_NAME}" -r '.VpcEndpoints[] | select(.Tags[0].Value == $name) | .VpcENDPOINT_ID')
15+
16+
if [[ "${#ENDPOINT_IDS[@]}" -eq 0 ]]; then
17+
echo "No endpoints to delete."
18+
break
19+
fi
20+
21+
echo "Deleting endpoints ${ENDPOINT_IDS[*]}..."
22+
for ENDPOINT_ID in "${ENDPOINT_IDS[@]}"; do
23+
echo "Deleting vpc endpoint ${ENDPOINT_ID}..."
24+
aws ec2 --region "${AWS_REGION}" delete-vpc-endpoints --vpc-endpoint-ids "${ENDPOINT_ID}"
25+
done
26+
27+
echo "Waiting 60 seconds..."
28+
sleep 60
29+
done
30+
echo "::endgroup::"
31+
32+
### Security Groups
33+
echo "::group::Deleting Security Groups..."
34+
mapfile -t SG_IDS < <(aws ec2 --region "${AWS_REGION}" describe-security-groups | jq --arg name "${SERVICE_NAME}" -r '.SecurityGroups[] | select(.Tags[0].Value == $name) | .GroupId')
35+
36+
if [[ "${#SG_IDS[@]}" -eq 0 ]]; then
37+
echo "No Security Groups to delete."
38+
fi
39+
40+
for SG_ID in "${SG_IDS[@]}"; do
41+
echo "Deleting SG ${SG_ID}..."
42+
aws ec2 --region "${AWS_REGION}" delete-security-group --group-id "${SG_ID}"
43+
done
44+
echo "::endgroup::"
45+
46+
### Subnets
47+
echo "::group::Deleting Subnets..."
48+
mapfile -t SUBNET_IDS < <(aws ec2 --region "${AWS_REGION}" describe-subnets | jq --arg name "${SERVICE_NAME}" -r '.Subnets[] | select(.Tags[0].Value == $name) | .SubnetId')
49+
50+
if [[ "${#SUBNET_IDS[@]}" -eq 0 ]]; then
51+
echo "No Subnets to delete."
52+
fi
53+
54+
for SUBNET_ID in "${SUBNET_IDS[@]}"; do
55+
echo "Deleting subnet ${SUBNET_ID}..."
56+
aws ec2 --region "${AWS_REGION}" delete-subnet --subnet-id "${SUBNET_ID}"
57+
done
58+
echo "::endgroup::"
59+
60+
### VPCS
61+
echo "::group::Deleting VPCs..."
62+
mapfile -t VPC_IDS < <(aws ec2 --region "${AWS_REGION}" describe-vpcs | jq --arg name "${SERVICE_NAME}" -r '.Vpcs[] | select(.Tags[0].Value == $name) | .VpcId')
63+
64+
if [[ "${#VPC_IDS[@]}" -eq 0 ]]; then
65+
echo "No VPCs to delete."
66+
fi
67+
68+
for VPC_ID in "${VPC_IDS[@]}"; do
69+
echo "Deleting vpc ${VPC_ID}..."
70+
aws ec2 --region "${AWS_REGION}" delete-vpc --vpc-id "${VPC_ID}"
71+
done
72+
echo "::endgroup::"

‎.github/actions/cleanup-clickhouse/action.yaml‎

Lines changed: 18 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,35 @@
1-
name: "Cleanup leftovers"
2-
description: "Cleanup any leftovers of e2e test"
1+
name: Cleanup leftovers
2+
description: Cleanup any leftovers of e2e test
33
inputs:
44
api_url:
55
required: false
6-
description: "Full URL of the API service to use. Defaults to the production API endpoint https://api.clickhouse.cloud/v1"
6+
description: Full URL of the API service to use. Defaults to the production API endpoint https://api.clickhouse.cloud/v1
77
default: ""
88
organization_id:
99
required: true
10-
description: "The clickhouse organization ID"
10+
description: The clickhouse organization ID
1111
token_key:
1212
required: true
13-
description: "The clickhouse token key"
13+
description: The clickhouse token key
14+
type: secret
1415
token_secret:
1516
required: true
16-
description: "The clickhouse token secret"
17+
description: The clickhouse token secret
18+
type: secret
1719
token:
1820
required: true
19-
description: "The unique token assigned to this e2e run"
21+
description: The unique token assigned to this e2e run
2022

2123
runs:
22-
using: "composite"
24+
using: composite
2325
steps:
24-
- name: cleanup clickhouse
26+
- name: Cleanup ClickHouse Instances
2527
shell: bash
28+
env:
29+
API_URL: ${{ inputs.api_url }}
30+
ORGANIZATION_ID: ${{ inputs.organization_id }}
31+
TOKEN_KEY: ${{ inputs.token_key }}
32+
TOKEN_SECRET: ${{ inputs.token_secret }}
33+
SUFFIX: ${{ inputs.token }}
2634
run: |
27-
api_url="${{ inputs.api_url }}"
28-
if [ "$api_url" == "" ]
29-
then
30-
api_url="https://api.clickhouse.cloud/v1"
31-
fi
32-
organization_id="${{ inputs.organization_id }}"
33-
token_key="${{ inputs.token_key }}"
34-
token_secret="${{ inputs.token_secret }}"
35-
suffix="${{ inputs.token }}"
36-
37-
echo "Deleting any service with suffix ${suffix}"
38-
39-
while :; do
40-
output="$(curl -su ${token_key}:${token_secret} ${api_url}/organizations/${organization_id}/services)"
41-
ids=$(echo "$output"|jq --arg suffix "${suffix}" -r '.result[]| select(.name | endswith($suffix)) |(.id + "," + .state)')
42-
43-
if [ "$ids" == "" ]
44-
then
45-
break
46-
fi
47-
48-
count="$(echo "$ids"|wc -l)"
49-
echo "There are ${count} services to be cleaned up"
50-
51-
for idandstatus in $ids
52-
do
53-
id="$(echo "${idandstatus}" | cut -d"," -f1)"
54-
status="$(echo "${idandstatus}" | cut -d"," -f2)"
55-
56-
case "$status" in
57-
stopped)
58-
echo "Deleting service ${id}"
59-
curl -su ${token_key}:${token_secret} -XDELETE "${api_url}/organizations/${organization_id}/services/${id}" -o /dev/null
60-
;;
61-
stopping)
62-
echo "Service ${id} is stopping, waiting"
63-
;;
64-
*)
65-
echo "Stopping service ${id}"
66-
curl -su ${token_key}:${token_secret} -XPATCH "${api_url}/organizations/${organization_id}/services/${id}/state" --data '{"command": "stop"}' -H 'Content-Type: application/json' -o /dev/null
67-
;;
68-
esac
69-
done
70-
71-
sleep 5
72-
done
73-
74-
echo "Cleanup complete"
35+
./.github/actions/cleanup-clickhouse/cleanup.sh

‎.github/actions/cleanup-clickhouse/cleanup.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
API_URL="${API_URL:-"https://api.clickhouse.cloud/v1"}"
6+
ORGANIZATION_ID="${ORGANIZATION_ID:?"ORGANIZATION_ID cannot be empty"}"
7+
TOKEN_KEY="${TOKEN_KEY:?"TOKEN_KEY cannot be empty"}"
8+
TOKEN_SECRET="${TOKEN_SECRET:?"TOKEN_SECRET cannot be empty"}"
9+
SUFFIX="${SUFFIX:?"SUFFIX cannot be empty"}"
10+
11+
echo "Deleting any service with suffix ${SUFFIX}..."
12+
13+
while :; do
14+
OUTPUT="$(curl -su "${TOKEN_KEY}:${TOKEN_SECRET}" "${API_URL}/organizations/${ORGANIZATION_ID}/services")"
15+
mapfile -t IDS < <(jq --arg suffix "${SUFFIX}" -r '.result[] | select(.name | contains($suffix)) | (.id + "," + .state)' <<<"${OUTPUT}")
16+
17+
if [[ "${#IDS[@]}" -eq 0 ]]; then
18+
echo "No services to cleanup."
19+
break
20+
fi
21+
22+
echo "There are ${#IDS[@]} services to be cleaned up."
23+
24+
for ID_AND_STATUS in "${IDS[@]}"; do
25+
ID="$(echo "${ID_AND_STATUS}" | cut -d"," -f1)"
26+
STATUS="$(echo "${ID_AND_STATUS}" | cut -d"," -f2)"
27+
28+
case "${STATUS}" in
29+
stopped)
30+
echo "Deleting service ${ID}..."
31+
curl -su "${TOKEN_KEY}:${TOKEN_SECRET}" -XDELETE "${API_URL}/organizations/${ORGANIZATION_ID}/services/${ID}" -o /dev/null
32+
;;
33+
stopping)
34+
echo "Service ${ID} is stopping, waiting..."
35+
;;
36+
*)
37+
echo "Stopping service ${ID}..."
38+
curl -su "${TOKEN_KEY}:${TOKEN_SECRET}" -XPATCH "${API_URL}/organizations/${ORGANIZATION_ID}/services/${ID}/state" --data '{"command": "stop"}' -H 'Content-Type: application/json' -o /dev/null
39+
;;
40+
esac
41+
done
42+
43+
echo "Waiting 5 seconds..."
44+
sleep 5
45+
done
46+
47+
echo "Cleanup complete."

‎.github/workflows/e2e.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ jobs:
6767
run: |
6868
# Number of seconds since January first 2025.
6969
token=$(( $(date +%s) - $(date -d "Jan 1, 2025 00:00:00" +%s) ))
70+
echo "token=${token}"
7071
echo "token=${token}" >> $GITHUB_OUTPUT
7172
7273
# Find the most recent release of terraform CLI
@@ -77,6 +78,7 @@ jobs:
7778
steps:
7879
- name: Checkout
7980
uses: actions/checkout@v4
81+
8082
- uses: ./.github/actions/find-tf-releases
8183
id: find
8284
with:
@@ -89,6 +91,7 @@ jobs:
8991
steps:
9092
- name: Checkout
9193
uses: actions/checkout@v4
94+
9295
- uses: ./.github/actions/list-examples
9396
id: list
9497
with:
@@ -114,6 +117,7 @@ jobs:
114117
uses: actions/checkout@v4
115118
with:
116119
fetch-depth: 0
120+
117121
- name: Generate test name
118122
id: name
119123
run: |
@@ -152,13 +156,15 @@ jobs:
152156
skip_build: "false"
153157
region: ${{ contains(fromJSON('["hipaa", "pci"]'), matrix.test.name) && steps.credentials.outputs.compliance_region || steps.credentials.outputs.region }}
154158
aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
159+
155160
- name: Cleanup
156161
if: ${{ always() && matrix.test.cloud == 'aws' }}
157162
uses: ./.github/actions/cleanup-aws
158163
with:
159164
service_name: ${{steps.name.outputs.test_name}}
160165
aws_region: ${{ steps.credentials.outputs.region }}
161166
aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
167+
162168
- name: Mark error
163169
id: status
164170
if: failure()

0 commit comments

Comments
 (0)