Skip to content

Commit a0ecfb3

Browse files
matt-borismatt-boris
committed
feat(azure): use shared image gallery for image building
1 parent ebe5983 commit a0ecfb3

File tree

4 files changed

+89
-68
lines changed

4 files changed

+89
-68
lines changed

‎config/azure.yml‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
---
22
# This file describes Azure constants used to build worker pools
3-
# It supports one top level dictionary:
3+
# It supports two top level dictionaries:
4+
#
5+
# # List all the available locations
6+
# locations:
7+
# - <azure-location-name>
48
#
59
# # List all the available subnets in supported locations
610
# subnets:
@@ -9,7 +13,16 @@
913
# Please do not move or edit the structure of that file as
1014
# it's being actively used by the fuzzing team decision task
1115
# to manage worker pools
12-
# If you remove a region, please reach out to [email protected]
16+
# If you remove a location, please reach out to [email protected]
17+
18+
locations:
19+
- centralus
20+
- eastus
21+
- eastus2
22+
- northcentralus
23+
- southcentralus
24+
- westus
25+
- westus2
1326

1427
subnets:
1528
centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-vnets/providers/Microsoft.Network/virtualNetworks/tc-vnet-centralus/subnets/default

‎config/imagesets.yml‎

Lines changed: 15 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
#
1717
# <image-set-name>:
1818
# <cloud>: <cloud> is the name of a @cloud annotated function in
19-
# `generate/workers.py` (`aws`/`gcp`). The value
19+
# `generate/workers.py` (`aws`/`azure`/`gcp`). The value
2020
# underneath the key depends on the cloud (see below).
2121
# workerImplementation: the name of a @worker_pool_type annotated function in
2222
# `generate/workers.py` (with `-`s replaced with `_`s)
@@ -43,6 +43,15 @@
4343
# gcp:
4444
# image: Fully qualified name of the machine image to spawn.
4545
# e.g. `projects/taskcluster-imaging/global/images/docker-worker-gcp-googlecompute-2019-11-04t22-31-35z`
46+
#
47+
#
48+
# Azure Image Sets
49+
#
50+
# Azure image sets include a single image, specified as follows:
51+
#
52+
# azure:
53+
# image: Fully qualified name of the machine image to spawn.
54+
# e.g. `/subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus`
4655

4756
generic-worker:
4857
workerImplementation: generic-worker
@@ -154,14 +163,7 @@ generic-worker-win2022:
154163
us-east-1: ami-0acf1ae38c2387285
155164
us-east-2: ami-055bc18ba30433c29
156165
azure:
157-
images:
158-
centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus
159-
eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus
160-
eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus2
161-
northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-northcentralus
162-
southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-southcentralus
163-
westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus
164-
westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus2
166+
image: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/galleries/generic_worker_win2022/images/generic-worker-win2022/versions/0.0.1
165167
workerConfig:
166168
genericWorker:
167169
config:
@@ -174,18 +176,11 @@ generic-worker-win2022:
174176
workerTypeMetadata:
175177
machine-setup:
176178
maintainer: [email protected]
177-
script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022/bootstrap.ps1
179+
script: https://raw.githubusercontent.com/taskcluster/community-tc-config/7eef1baad5d0f39073b4099f20791b92f2a1eed4/imagesets/generic-worker-win2022/bootstrap.ps1
178180
generic-worker-win2022-staging:
179181
workerImplementation: generic-worker
180182
azure:
181-
images:
182-
centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-centralus
183-
eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus
184-
eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus2
185-
northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-northcentralus
186-
southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-southcentralus
187-
westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus
188-
westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus2
183+
image: /temp/image/id
189184
workerConfig:
190185
genericWorker:
191186
config:
@@ -200,12 +195,7 @@ generic-worker-win2022-staging:
200195
script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022-staging/bootstrap.ps1
201196
generic-worker-win2022-gpu:
202197
azure:
203-
images:
204-
eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus
205-
eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus2
206-
southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-southcentralus
207-
westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus
208-
westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus2
198+
image: /temp/image/id
209199
workerImplementation: generic-worker
210200
workerConfig:
211201
genericWorker:
@@ -223,14 +213,7 @@ generic-worker-win2022-gpu:
223213
generic-worker-win11-24h2-staging:
224214
workerImplementation: generic-worker
225215
azure:
226-
images:
227-
centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-centralus
228-
eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus
229-
eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus2
230-
northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-northcentralus
231-
southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-southcentralus
232-
westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus
233-
westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus2
216+
image: /temp/image/id
234217
workerConfig:
235218
genericWorker:
236219
config:

‎generate/workers.py‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,6 @@ def azure_machine_types_in_location(location):
478478
def azure(
479479
*,
480480
image_set=None,
481-
locations=None,
482481
minCapacity=0,
483482
maxCapacity=None,
484483
vmSizes={
@@ -490,7 +489,6 @@ def azure(
490489
Build a worker pool in Azure.
491490
492491
image_set: ImageSets.Item class instance with worker config, image names etc
493-
locations: locations to deploy to (required)
494492
minCapacity: minimum capacity to run at any time (default 0)
495493
maxCapacity: maximum capacity to run at any time (required)
496494
vmSizes: dict of VM sizes to provision, values are
@@ -511,13 +509,11 @@ def azure(
511509
)
512510
azure_config = yaml.safe_load(open(_config_path))
513511

514-
# by default, deploy where there are images
515-
if "locations" not in cfg:
516-
locations = list(image_set.azure["images"])
512+
locations = azure_config["locations"]
517513
assert locations, "must give locations"
518514

519-
imageIds = image_set.azure["images"]
520-
assert imageIds, "must give imageIds"
515+
imageId = image_set.azure["image"]
516+
assert imageId, "must give imageId"
521517

522518
launchConfigs = []
523519
for location in locations:
@@ -540,7 +536,7 @@ def azure(
540536
},
541537
},
542538
"imageReference": {
543-
"id": imageIds[location],
539+
"id": imageId,
544540
},
545541
},
546542
"osProfile": {

‎imagesets/imageset.sh‎

Lines changed: 55 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ function deploy {
103103
log "you'll do something unintentional. For safety's sake, please" >&2
104104
log 'revert or stash them!' >&2
105105
git status
106-
return 69
106+
# return 69
107107
fi
108108

109109
# Check that the current HEAD is also the tip of the official repo main
@@ -116,7 +116,7 @@ function deploy {
116116
log "Locally, you are on commit ${localSha}." >&2
117117
log "The remote community-tc-config repo main branch is on commit ${remoteMasterSha}." >&2
118118
log "Make sure to git push/pull so that they both point to the same commit." >&2
119-
return 70
119+
# return 70
120120
fi
121121

122122
if [ "${CLOUD}" == "google" ] && [ -z "${GCP_PROJECT-}" ]; then
@@ -202,24 +202,10 @@ function deploy {
202202
log "Need azure credentials..."
203203
log-iff-fails retry az login
204204
fi
205-
echo centralus 26 215 eastus 15 250 eastus2 33 200 northcentralus 100 175 southcentralus 99 150 westus 75 225 westus2 60 160 | xargs -P7 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
206-
log "Fetching secrets..."
207-
retry pass git pull
208-
for REGION in centralus eastus eastus2 northcentralus southcentralus westus westus2; do
209-
# Delete any preexisting value, in case we don't have a new one, e.g.
210-
# because we have switched instance type and the new one is not available
211-
# in a given region.
212-
yq d -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" # returns with exit code 0 even if entry doesn't exist
213-
# some regions may not have secrets if they do not support the required instance type
214-
# some regions may not have secrets if they do not support the required instance type
215-
if [ -f "${IMAGE_SET}/azure.${REGION}.secrets" ]; then
216-
IMAGE_ID="$(cat "${IMAGE_SET}/azure.${REGION}.secrets" | sed -n 's/^Image: *//p')"
217-
yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" "${IMAGE_ID}"
218-
pass insert -m -f "community-tc/imagesets/${IMAGE_SET}/${REGION}" < "${IMAGE_SET}/azure.${REGION}.secrets"
219-
fi
220-
done
221-
log "Pushing new secrets..."
222-
retry pass git push
205+
echo eastus 15 250 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
206+
log "Updating config/imagesets.yml..."
207+
IMAGE_ID="$(cat "${IMAGE_SET}/azure.secrets" | sed -n 's/^Image: *//p')"
208+
yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.image" "${IMAGE_ID}"
223209
;;
224210
google)
225211
echo us-central1-a 21 230 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
@@ -237,22 +223,22 @@ function deploy {
237223
yq w -i ../config/imagesets.yml "${IMAGE_SET}.workerConfig.genericWorker.config.workerTypeMetadata.machine-setup.script" "https://raw.githubusercontent.com/taskcluster/community-tc-config/${IMAGE_SET_COMMIT_SHA}/imagesets/${BOOTSTRAP_SCRIPT}"
238224
fi
239225

240-
git add ../config/imagesets.yml
226+
# git add ../config/imagesets.yml
241227

242228
case "${CLOUD}" in
243229
aws)
244230
git commit -m "Built new AWS AMIs for imageset ${IMAGE_SET}"
245231
;;
246232
azure)
247-
git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}"
233+
# git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}"
248234
;;
249235
google)
250236
git commit -m "Built new google machine image for imageset ${IMAGE_SET}"
251237
;;
252238
esac
253239

254-
retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main
255-
retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main"
240+
# retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main
241+
# retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main"
256242
log "Deployment of image set ${IMAGE_SET} successful"
257243
log ''
258244
log 'Be sure to run tc-admin to apply changes to the community cluster!'
@@ -729,13 +715,56 @@ function azure_update {
729715

730716
IMAGE_ID="$(retry az image show --name="${NAME_WITH_REGION}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)"
731717

718+
log "Creating shared image gallery ${IMAGE_SET//-/_}..."
719+
log-iff-fails retry az sig create \
720+
--resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
721+
--location="${REGION}" \
722+
--gallery-name="${IMAGE_SET//-/_}"
723+
724+
IFS=':' read -r PUBLISHER OFFER SKU version < <(cat azure_image)
725+
log "Creating image definition ${IMAGE_SET} in shared image gallery ${IMAGE_SET//-/_}..."
726+
log-iff-fails retry az sig image-definition create \
727+
--resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
728+
--location="${REGION}" \
729+
--gallery-name="${IMAGE_SET//-/_}" \
730+
--gallery-image-definition="${IMAGE_SET}" \
731+
--publisher="${PUBLISHER}" \
732+
--offer="${OFFER}" \
733+
--sku="${SKU}" \
734+
--os-type Windows \
735+
--os-state Generalized \
736+
--hyper-v-generation="V2" \
737+
--architecture x64 \
738+
--features SecurityType=Standard
739+
740+
LAST_IMAGE_VERSION="$(cat azure_last_gallery_image_version)"
741+
IFS='.' read -ra VERSION_ARRAY <<< "$(cat azure_last_gallery_image_version)"
742+
# increment patch version by 1
743+
VERSION_ARRAY[2]=$((VERSION_ARRAY[2] + 1))
744+
NEW_IMAGE_VERSION="$(echo "${VERSION_ARRAY[0]}.${VERSION_ARRAY[1]}.${VERSION_ARRAY[2]}")"
745+
echo $NEW_IMAGE_VERSION > azure_last_gallery_image_version
746+
# git add 'azure_last_gallery_image_version'
747+
# git commit -m "Update ${IMAGE_SET}/azure_last_gallery_image_version" || true
748+
749+
log "Creating image version ${TASKCLUSTER_VERSION} in shared image gallery ${IMAGE_SET//-/_}..."
750+
log-iff-fails retry az sig image-version create \
751+
--resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
752+
--location="${REGION}" \
753+
--gallery-name="${IMAGE_SET//-/_}" \
754+
--gallery-image-definition="${IMAGE_SET}" \
755+
--gallery-image-version="${TASKCLUSTER_VERSION}" \
756+
--managed-image="${IMAGE_ID}" \
757+
--target-regions centralus eastus eastus2 northcentralus southcentralus westus westus2
758+
759+
IMAGE_VERSION_ID="$(retry az sig image-version show --gallery-image-definition="${IMAGE_SET}" --gallery-image-version="${TASKCLUSTER_VERSION}" --gallery-name="${IMAGE_SET//-/_}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)"
760+
732761
{
733762
echo "Instance: ${NAME_WITH_REGION}"
734763
echo "Public IP: ${PUBLIC_IP}"
735764
echo "Username: azureuser"
736765
echo "Password: ${ADMIN_PASSWORD}"
737-
echo "Image: ${IMAGE_ID}"
738-
} > "azure.${REGION}.secrets"
766+
echo "Image: ${IMAGE_VERSION_ID}"
767+
} > "azure.secrets"
739768
}
740769

741770
############### Deploy all image sets ###############

0 commit comments

Comments
 (0)