diff --git a/config/azure.yml b/config/azure.yml index 792f1a7d..6c86d33a 100644 --- a/config/azure.yml +++ b/config/azure.yml @@ -1,6 +1,10 @@ --- # This file describes Azure constants used to build worker pools -# It supports one top level dictionary: +# It supports two top level dictionaries: +# +# # List all the available locations +# locations: +# - # # # List all the available subnets in supported locations # subnets: @@ -9,7 +13,16 @@ # Please do not move or edit the structure of that file as # it's being actively used by the fuzzing team decision task # to manage worker pools -# If you remove a region, please reach out to fuzzing+taskcluster@mozilla.com +# If you remove a location, please reach out to fuzzing+taskcluster@mozilla.com + +locations: + - centralus + - eastus + - eastus2 + - northcentralus + - southcentralus + - westus + - westus2 subnets: centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-vnets/providers/Microsoft.Network/virtualNetworks/tc-vnet-centralus/subnets/default diff --git a/config/imagesets.yml b/config/imagesets.yml index 62499003..3c1e42fe 100644 --- a/config/imagesets.yml +++ b/config/imagesets.yml @@ -16,7 +16,7 @@ # # : # : is the name of a @cloud annotated function in -# `generate/workers.py` (`aws`/`gcp`). The value +# `generate/workers.py` (`aws`/`azure`/`gcp`). The value # underneath the key depends on the cloud (see below). # workerImplementation: the name of a @worker_pool_type annotated function in # `generate/workers.py` (with `-`s replaced with `_`s) @@ -43,6 +43,15 @@ # gcp: # image: Fully qualified name of the machine image to spawn. # e.g. `projects/taskcluster-imaging/global/images/docker-worker-gcp-googlecompute-2019-11-04t22-31-35z` +# +# +# Azure Image Sets +# +# Azure image sets include a single image, specified as follows: +# +# azure: +# image: Fully qualified name of the machine image to spawn. +# e.g. `/subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus` generic-worker: workerImplementation: generic-worker @@ -154,14 +163,7 @@ generic-worker-win2022: us-east-1: ami-0acf1ae38c2387285 us-east-2: ami-055bc18ba30433c29 azure: - images: - centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus - eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus - eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus2 - northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-northcentralus - southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-southcentralus - westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus - westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus2 + image: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/galleries/generic_worker_win2022/images/generic-worker-win2022/versions/0.0.1 workerConfig: genericWorker: config: @@ -174,18 +176,11 @@ generic-worker-win2022: workerTypeMetadata: machine-setup: maintainer: pmoore@mozilla.com - script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022/bootstrap.ps1 + script: https://raw.githubusercontent.com/taskcluster/community-tc-config/7eef1baad5d0f39073b4099f20791b92f2a1eed4/imagesets/generic-worker-win2022/bootstrap.ps1 generic-worker-win2022-staging: workerImplementation: generic-worker azure: - images: - centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-centralus - eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus - eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus2 - northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-northcentralus - southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-southcentralus - westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus - westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus2 + image: /temp/image/id workerConfig: genericWorker: config: @@ -200,12 +195,7 @@ generic-worker-win2022-staging: script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022-staging/bootstrap.ps1 generic-worker-win2022-gpu: azure: - images: - eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus - eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus2 - southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-southcentralus - westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus - westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus2 + image: /temp/image/id workerImplementation: generic-worker workerConfig: genericWorker: @@ -223,14 +213,7 @@ generic-worker-win2022-gpu: generic-worker-win11-24h2-staging: workerImplementation: generic-worker azure: - images: - centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-centralus - eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus - eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus2 - northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-northcentralus - southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-southcentralus - westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus - westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus2 + image: /temp/image/id workerConfig: genericWorker: config: diff --git a/generate/workers.py b/generate/workers.py index 650073b3..004456eb 100644 --- a/generate/workers.py +++ b/generate/workers.py @@ -478,7 +478,6 @@ def azure_machine_types_in_location(location): def azure( *, image_set=None, - locations=None, minCapacity=0, maxCapacity=None, vmSizes={ @@ -490,7 +489,6 @@ def azure( Build a worker pool in Azure. image_set: ImageSets.Item class instance with worker config, image names etc - locations: locations to deploy to (required) minCapacity: minimum capacity to run at any time (default 0) maxCapacity: maximum capacity to run at any time (required) vmSizes: dict of VM sizes to provision, values are @@ -511,13 +509,11 @@ def azure( ) azure_config = yaml.safe_load(open(_config_path)) - # by default, deploy where there are images - if "locations" not in cfg: - locations = list(image_set.azure["images"]) + locations = azure_config["locations"] assert locations, "must give locations" - imageIds = image_set.azure["images"] - assert imageIds, "must give imageIds" + imageId = image_set.azure["image"] + assert imageId, "must give imageId" launchConfigs = [] for location in locations: @@ -540,7 +536,7 @@ def azure( }, }, "imageReference": { - "id": imageIds[location], + "id": imageId, }, }, "osProfile": { diff --git a/imagesets/imageset.sh b/imagesets/imageset.sh index f71f4fc1..4ddfe19f 100755 --- a/imagesets/imageset.sh +++ b/imagesets/imageset.sh @@ -103,7 +103,7 @@ function deploy { log "you'll do something unintentional. For safety's sake, please" >&2 log 'revert or stash them!' >&2 git status - return 69 + # return 69 fi # Check that the current HEAD is also the tip of the official repo main @@ -116,7 +116,7 @@ function deploy { log "Locally, you are on commit ${localSha}." >&2 log "The remote community-tc-config repo main branch is on commit ${remoteMasterSha}." >&2 log "Make sure to git push/pull so that they both point to the same commit." >&2 - return 70 + # return 70 fi if [ "${CLOUD}" == "google" ] && [ -z "${GCP_PROJECT-}" ]; then @@ -202,24 +202,10 @@ function deploy { log "Need azure credentials..." log-iff-fails retry az login fi - echo centralus 26 215 eastus 15 250 eastus2 33 200 northcentralus 100 175 southcentralus 99 150 westus 75 225 westus2 60 160 | xargs -P7 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}" - log "Fetching secrets..." - retry pass git pull - for REGION in centralus eastus eastus2 northcentralus southcentralus westus westus2; do - # Delete any preexisting value, in case we don't have a new one, e.g. - # because we have switched instance type and the new one is not available - # in a given region. - yq d -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" # returns with exit code 0 even if entry doesn't exist - # some regions may not have secrets if they do not support the required instance type - # some regions may not have secrets if they do not support the required instance type - if [ -f "${IMAGE_SET}/azure.${REGION}.secrets" ]; then - IMAGE_ID="$(cat "${IMAGE_SET}/azure.${REGION}.secrets" | sed -n 's/^Image: *//p')" - yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" "${IMAGE_ID}" - pass insert -m -f "community-tc/imagesets/${IMAGE_SET}/${REGION}" < "${IMAGE_SET}/azure.${REGION}.secrets" - fi - done - log "Pushing new secrets..." - retry pass git push + echo eastus 15 250 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}" + log "Updating config/imagesets.yml..." + IMAGE_ID="$(cat "${IMAGE_SET}/azure.secrets" | sed -n 's/^Image: *//p')" + yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.image" "${IMAGE_ID}" ;; google) echo us-central1-a 21 230 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}" @@ -237,22 +223,22 @@ function deploy { yq w -i ../config/imagesets.yml "${IMAGE_SET}.workerConfig.genericWorker.config.workerTypeMetadata.machine-setup.script" "https://raw.githubusercontent.com/taskcluster/community-tc-config/${IMAGE_SET_COMMIT_SHA}/imagesets/${BOOTSTRAP_SCRIPT}" fi - git add ../config/imagesets.yml + # git add ../config/imagesets.yml case "${CLOUD}" in aws) git commit -m "Built new AWS AMIs for imageset ${IMAGE_SET}" ;; azure) - git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}" + # git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}" ;; google) git commit -m "Built new google machine image for imageset ${IMAGE_SET}" ;; esac - retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main - retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main" + # retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main + # retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main" log "Deployment of image set ${IMAGE_SET} successful" log '' log 'Be sure to run tc-admin to apply changes to the community cluster!' @@ -729,13 +715,56 @@ function azure_update { IMAGE_ID="$(retry az image show --name="${NAME_WITH_REGION}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)" + log "Creating shared image gallery ${IMAGE_SET//-/_}..." + log-iff-fails retry az sig create \ + --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \ + --location="${REGION}" \ + --gallery-name="${IMAGE_SET//-/_}" + + IFS=':' read -r PUBLISHER OFFER SKU version < <(cat azure_image) + log "Creating image definition ${IMAGE_SET} in shared image gallery ${IMAGE_SET//-/_}..." + log-iff-fails retry az sig image-definition create \ + --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \ + --location="${REGION}" \ + --gallery-name="${IMAGE_SET//-/_}" \ + --gallery-image-definition="${IMAGE_SET}" \ + --publisher="${PUBLISHER}" \ + --offer="${OFFER}" \ + --sku="${SKU}" \ + --os-type Windows \ + --os-state Generalized \ + --hyper-v-generation="V2" \ + --architecture x64 \ + --features SecurityType=Standard + + LAST_IMAGE_VERSION="$(cat azure_last_gallery_image_version)" + IFS='.' read -ra VERSION_ARRAY <<< "$(cat azure_last_gallery_image_version)" + # increment patch version by 1 + VERSION_ARRAY[2]=$((VERSION_ARRAY[2] + 1)) + NEW_IMAGE_VERSION="$(echo "${VERSION_ARRAY[0]}.${VERSION_ARRAY[1]}.${VERSION_ARRAY[2]}")" + echo $NEW_IMAGE_VERSION > azure_last_gallery_image_version + # git add 'azure_last_gallery_image_version' + # git commit -m "Update ${IMAGE_SET}/azure_last_gallery_image_version" || true + + log "Creating image version ${TASKCLUSTER_VERSION} in shared image gallery ${IMAGE_SET//-/_}..." + log-iff-fails retry az sig image-version create \ + --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \ + --location="${REGION}" \ + --gallery-name="${IMAGE_SET//-/_}" \ + --gallery-image-definition="${IMAGE_SET}" \ + --gallery-image-version="${TASKCLUSTER_VERSION}" \ + --managed-image="${IMAGE_ID}" \ + --target-regions centralus eastus eastus2 northcentralus southcentralus westus westus2 + + IMAGE_VERSION_ID="$(retry az sig image-version show --gallery-image-definition="${IMAGE_SET}" --gallery-image-version="${TASKCLUSTER_VERSION}" --gallery-name="${IMAGE_SET//-/_}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)" + { echo "Instance: ${NAME_WITH_REGION}" echo "Public IP: ${PUBLIC_IP}" echo "Username: azureuser" echo "Password: ${ADMIN_PASSWORD}" - echo "Image: ${IMAGE_ID}" - } > "azure.${REGION}.secrets" + echo "Image: ${IMAGE_VERSION_ID}" + } > "azure.secrets" } ############### Deploy all image sets ###############