diff --git a/hack/cache-common-inc.sh b/hack/cache-common-inc.sh index f5bfbfc9796..74fdd059764 100755 --- a/hack/cache-common-inc.sh +++ b/hack/cache-common-inc.sh @@ -216,6 +216,26 @@ function location_to_sha256() { ) } +# e.g. +# ```console +# $ cache_download_dir +# .download # on GitHub Actions +# /home/user/.cache/lima/download # on Linux +# /Users/user/Library/Caches/lima/download # on macOS +# /home/user/.cache/lima/download # on others +# ``` +function cache_download_dir() { + if [[ ${GITHUB_ACTIONS:-false} == true ]]; then + echo ".download" + else + case "$(uname -s)" in + Linux) echo "${XDG_CACHE_HOME:-${HOME}/.cache}/lima/download" ;; + Darwin) echo "${HOME}/Library/Caches/lima/download" ;; + *) echo "${HOME}/.cache/lima/download" ;; + esac + fi +} + # e.g. # ```console # $ location_to_cache_path "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" @@ -224,7 +244,7 @@ function location_to_sha256() { function location_to_cache_path() { local location=$1 [[ ${location} != "null" ]] || return - sha256=$(location_to_sha256 "${location}") && echo ".download/by-url-sha256/${sha256}" + sha256=$(location_to_sha256 "${location}") && download_dir=$(cache_download_dir) && echo "${download_dir}/by-url-sha256/${sha256}" } # e.g. @@ -321,3 +341,48 @@ function hash_file() { echo "${hash}" | xxd -r -p | sha256sum | cut -d' ' -f1 ) } + +# Download the file to the cache directory and print the path. +# e.g. +# ```console +# $ download_to_cache "https://cloud-images.ubuntu.com/releases/24.04/release-20240821/ubuntu-24.04-server-cloudimg-arm64.img" +# .download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on GitHub Actions +# /home/user/.cache/lima/download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on Linux +# /Users/user/Library/Caches/lima/download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on macOS +# /home/user/.cache/lima/download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on others +function download_to_cache() { + local code_time_type_url + code_time_type_url=$( + curl -sSLI -w "%{http_code}\t%header{Last-Modified}\t%header{Content-Type}\t%{url_effective}" "$1" -o /dev/null + ) + + local code time type url + IFS=$'\t' read -r code time type url filename <<<"${code_time_type_url}" + [[ ${code} == 200 ]] || exit 1 + + local cache_path + cache_path=$(location_to_cache_path "${url}") + [[ -d ${cache_path} ]] || mkdir -p "${cache_path}" + + local needs_download=0 + [[ -f ${cache_path}/data ]] || needs_download=1 + [[ -f ${cache_path}/time && "$(<"${cache_path}/time")" == "${time}" ]] || needs_download=1 + [[ -f ${cache_path}/type && "$(<"${cache_path}/type")" == "${type}" ]] || needs_download=1 + if [[ ${needs_download} -eq 1 ]]; then + local code_time_type_url_filename + code_time_type_url_filename=$( + echo "downloading ${url}" >&2 + curl -SL -w "%{http_code}\t%header{Last-Modified}\t%header{Content-Type}\t%{url_effective}\t%{filename_effective}" --no-clobber -o "${cache_path}/data" "${url}" + ) + local filename + IFS=$'\t' read -r code time type url filename <<<"${code_time_type_url_filename}" + [[ ${code} == 200 ]] || exit 1 + [[ "${cache_path}/data" == "${filename}" ]] || mv "${filename}" "${cache_path}/data" + # sha256.digest seems existing if expected digest is available. so, not creating it here. + # sha256sum "${cache_path}/data" | awk '{print "sha256:"$1}' >"${cache_path}/sha256.digest" + echo -n "${time}" >"${cache_path}/time" + fi + [[ -f ${cache_path}/type ]] || echo -n "${type}" >"${cache_path}/type" + [[ -f ${cache_path}/url ]] || echo -n "${url}" >"${cache_path}/url" + echo "${cache_path}/data" +} diff --git a/hack/update-template-ubuntu.sh b/hack/update-template-ubuntu.sh new file mode 100755 index 00000000000..082c1169618 --- /dev/null +++ b/hack/update-template-ubuntu.sh @@ -0,0 +1,335 @@ +#!/usr/bin/env bash + +function print_help() { + cat <|--minimal|--server] [--version ] ... + +Description: + This script updates the Ubuntu image location in the specified templates. + If the image location in the template contains a release date in the URL, the script replaces it with the latest available date. + If no flags are specified, the script uses the flavor and version from the image location basename in the template. + + Image location basename format: ubuntu---cloudimg-.img + + Released Ubuntu image information is fetched from the following URLs: + + Server: https://cloud-images.ubuntu.com/releases/stream/v1/com.ubuntu.cloud:released:download.json + Minimal: https://cloud-images.ubuntu.com/minimal/releases/stream/v1/com.ubuntu.cloud:released:download.json + + The downloaded JSON file will be cached in the Lima cache directory. + +Examples: + Update the Ubuntu image location in templates/**.yaml: + $ $(basename "${BASH_SOURCE[0]}") templates/**.yaml + + Update the Ubuntu image location in ~/.lima/ubuntu/lima.yaml: + $ $(basename "${BASH_SOURCE[0]}") ~/.lima/ubuntu/lima.yaml + + Update the Ubuntu image location to ubuntu-24.04-minimal-cloudimg-.img in ~/.lima/docker/lima.yaml: + $ $(basename "${BASH_SOURCE[0]}") --minimal --version 24.04 ~/.lima/docker/lima.yaml + +Flags: + --flavor Use the specified flavor image + --server Shortcut for --flavor server + --minimal Shortcut for --flavor minimal + --version Use the specified version + -h, --help Print this help message +HELP +} + +scriptdir=$(dirname "${BASH_SOURCE[0]}") +# shellcheck source=./cache-common-inc.sh +# shellcheck disable=SC1091 +. "${scriptdir}/cache-common-inc.sh" + +set -eu -o pipefail + +readonly -A base_urls=( + [minimal]=https://cloud-images.ubuntu.com/minimal/releases/ + [server]=https://cloud-images.ubuntu.com/releases/ +) + +# validate_url checks if the URL is valid and returns the location if it is. +# If the URL is redirected, it returns the redirected location. +# e.g. +# ```console +# validate_url https://cloud-images.ubuntu.com/server/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img +# https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img +# ``` +function validate_url() { + local url=$1 + code_location=$(curl -sSL -o /dev/null -I -w "%{http_code}\t%{url_effective}" "${url}") + read -r code location <<<"${code_location}" + [[ ${code} -eq 200 ]] && echo "${location}" +} + +# ubuntu_base_url returns the base URL for the given flavor. +# e.g. +# ```console +# ubuntu_base_url minimal +# https://cloud-images.ubuntu.com/minimal/releases/ +# ``` +function ubuntu_base_url() { + # shellcheck disable=SC2015 + [[ -v base_urls[$1] ]] && echo "${base_urls[$1]}" || ( + echo "Unsupported flavor: $1" >&2 + exit 1 + ) +} + +# downloaded_json downloads the JSON file for the given flavor and returns the path. +# e.g. +# ```console +# downloaded_json server +# /Users/user/Library/Caches/lima/download/by-url-sha256/255f982f5bbda07f5377369093e21c506d7240f5ba901479bdadfa205ddafb01/data +# ``` +function downloaded_json() { + local flavor=$1 base_url json_url + json_url=$(ubuntu_base_url "${flavor}")streams/v1/com.ubuntu.cloud:released:download.json + download_to_cache "${json_url}" +} + +# ubuntu_image_url_try_replace_release_with_version tries to replace the release with the version in the URL. +# If the URL is valid, it returns the URL with the version. +function ubuntu_image_url_try_replace_release_with_version() { + local location=$1 release=$2 version=$3 location_using_version + # shellcheck disable=SC2310 + if location_using_version=$(validate_url "${location/\/${release}\//\/${version}\/}"); then + echo "${location_using_version}" + else + echo "${location}" + fi +} + +# ubuntu_image_url_latest returns the latest image URL and its digest for the given version, flavor, arch, and path suffix. +function ubuntu_image_url_latest() { + local version=$1 flavor=$2 arch=$3 path_suffix=$4 base_url downloaded_json jq_filter location_digest_release + base_url=$(ubuntu_base_url "${flavor}") + # shellcheck disable=SC2310 + downloaded_json=$(downloaded_json "${flavor}") || return 0 + jq_filter=" + [ + .products[\"com.ubuntu.cloud:${flavor}:${version}:${arch}\"] | + .release as \$release | + .versions[]?.items[] | select(.path | endswith(\"${path_suffix}\")) | + [\"${base_url}\"+.path, \"sha256:\"+.sha256, \$release] | @tsv + ] | last + " + location_digest_release=$(jq -e -r "${jq_filter}" "${downloaded_json}") || return 0 + local location digest release location_using_version + read -r location digest release <<<"${location_digest_release}" + # shellcheck disable=SC2310 + location=$(validate_url "${location}") || return 0 + location=$(ubuntu_image_url_try_replace_release_with_version "${location}" "${release}" "${version}") + echo -e "${location}\t${digest}" +} + +# ubuntu_image_url_release returns the release image URL for the given version, flavor, arch, and path suffix. +function ubuntu_image_url_release() { + local version=$1 flavor=$2 arch=$3 path_suffix=$4 base_url + base_url=$(ubuntu_base_url "${flavor}") + # shellcheck disable=SC2310 + downloaded_json=$(downloaded_json "${flavor}") || return 0 + local location release location_using_version + jq_filter=" + [ + .products | to_entries[] as \$product_entry | + \$product_entry.value| select(.version == \"${version}\") | + .release + ] | first + " + release=$(jq -e -r "${jq_filter}" "${downloaded_json}") || return 0 + # shellcheck disable=SC2310 + location=$(validate_url "${base_url}${release}/release/ubuntu-${version}-${flavor}-cloudimg-${arch}${path_suffix}") || return 0 + ubuntu_image_url_try_replace_release_with_version "${location}" "${release}" "${version}" +} + +# ubuntu_kernel_info_for_image_url returns the kernel and initrd location and digest for the given location. +function ubuntu_kernel_info_for_image_url() { + local location=$1 location_dirname sha256sums location_basename + location_dirname=$(dirname "${location}")/unpacked + sha256sums=$(curl -sSLf "${location_dirname}/SHA256SUMS") + location_basename="$(basename "${location}" | cut -d- -f1-5 | cut -d. -f1-2)" + + # kernel + local kernel_basename kernel_location kernel_digest + kernel_basename="${location_basename}-vmlinuz-generic" + # shellcheck disable=SC2310 + kernel_location=$(validate_url "${location_dirname}/${kernel_basename}") || return 0 + kernel_digest=${kernel_location+$(awk "/${kernel_basename}/{print \"sha256:\"\$1}" <<<"${sha256sums}")} + + # initrd + local initrd_basename initrd_location initrd_digest + initrd_basename="${location_basename}-initrd-generic" + initrd_location=$(validate_url "${location_dirname}/${initrd_basename}") + initrd_digest=${initrd_location+$(awk "/${initrd_basename}/{print \"sha256:\"\$1}" <<<"${sha256sums}")} + + echo -e "${kernel_location}\t${kernel_digest}\t${initrd_location}\t${initrd_digest}" +} + +# limayaml_arch returns the arch in the lima.yaml format +function limayaml_arch() { + local arch=$1 + arch=${arch/amd64/x86_64} + arch=${arch/arm64/aarch64} + arch=${arch/armhf/armv7l} + echo "${arch}" +} + +declare -a templates=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + print_help + exit 0 + ;; + --flavor) + if [[ -n $2 && $2 != -* ]]; then + flavor="$2" + shift + else + echo "Error: --flavor requires a value" >&2 + exit 1 + fi + ;; + --flavor=*) flavor="${1#*=}" ;; + --minimal) flavor="minimal" ;; + --server) flavor="server" ;; + --version) + if [[ -n $2 && $2 != -* ]]; then + version="$2" + shift + else + echo "Error: --version requires a value" >&2 + exit 1 + fi + ;; + --version=*) version="${1#*=}" ;; + *.yaml) templates+=("$1") ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac + shift +done + +if [[ ${#templates[@]} -eq 0 ]]; then + print_help + exit 0 +fi + +flavor=${flavor:-server} +downloaded_json=$(downloaded_json "${flavor}") +version="${version:-$(jq -r '[.products[]|.version|select(endswith(".04"))]|last' "${downloaded_json}")}" + +declare -A ubuntu_image_url_latest_cache=() +declare -A ubuntu_image_url_release_cache=() + +for template in "${templates[@]}"; do + echo "Processing ${template}" + # 1. extract location by parsing template using arch + yq_filter=" + .images[] | [.location, .kernel.location, .kernel.cmdline, .initrd.location] | @tsv + " + parsed=$(yq eval "${yq_filter}" "${template}") + + # 3. get the image location + arr=() + while IFS= read -r line; do arr+=("${line}"); done <<<"${parsed}" + locations=("${arr[@]}") + for ((index = 0; index < ${#locations[@]}; index++)); do + [[ ${locations[index]} != "null" ]] || continue + IFS=$'\t' read -r location kernel_location kernel_cmdline initrd_location <<<"${locations[index]}" + location_before="${location}" + + case "${location}" in + https://cloud-images.ubuntu.com/minimal/releases/*/release/*) use_latest=0 ;;& + https://cloud-images.ubuntu.com/minimal/releases/*/release-*/*) use_latest=1 ;;& + https://cloud-images.ubuntu.com/minimal/releases/*) flavor=${flavor:-minimal} ;; + https://cloud-images.ubuntu.com/releases/*/release/*) use_latest=0 ;;& + https://cloud-images.ubuntu.com/releases/*/release-*/*) use_latest=1 ;;& + https://cloud-images.ubuntu.com/releases/*) flavor=${flavor:-server} ;; + *) + # echo "Unsupported image location: ${location}" >&2 + continue + ;; + esac + + location_basename=$(basename "${location}") + version=${version:-$(echo "${location_basename}" | cut -d- -f2)} + flavor=${flavor:-$(echo "${location_basename}" | cut -d- -f3)} + arch=$(echo "${location_basename}" | cut -d- -f5 | cut -d. -f1) + path_suffix="${location_basename##*"${arch}"}" + limayaml_arch=$(limayaml_arch "${arch}") + if [[ ${use_latest} -eq 1 ]]; then + latest_cache_key=${version}-${flavor}-${arch}-${path_suffix} + location_digest=$( + # shellcheck disable=SC2015 + [[ -v ubuntu_image_url_latest_cache[${latest_cache_key}] ]] && echo "${ubuntu_image_url_latest_cache[${latest_cache_key}]}" || + ubuntu_image_url_latest "${version}" "${flavor}" "${arch}" "${path_suffix}" + ) + ubuntu_image_url_latest_cache[${latest_cache_key}]="${location_digest}" + read -r location digest <<<"${location_digest}" + if [[ -z ${location} ]]; then + echo "Failed to get the latest image location for ${location_basename}" >&2 + continue + elif [[ ${location} == "${location_before}" ]]; then + continue + fi + image_entry="{\"location\": \"${location}\", \"arch\": \"${limayaml_arch}\", \"digest\": \"${digest}\"}" + echo -e "${location}\n${digest}" + if [[ ${kernel_location} != "null" ]]; then + kernel_info=$(ubuntu_kernel_info_for_image_url "${location}") + IFS=$'\t' read -r kernel_location kernel_digest initrd_location initrd_digest <<<"${kernel_info}" + if [[ -n ${kernel_location} ]]; then + image_entry=$(jq ". + {kernel: {location: \"${kernel_location}\", digest: \"${kernel_digest}\"}}" <<<"${image_entry}") + [[ ${kernel_cmdline} != "null" ]] && image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") + echo -e "${kernel_location}\n${kernel_digest}" + fi + if [[ -n ${initrd_location} ]]; then + image_entry=$(jq ". + {initrd: {location: \"${initrd_location}\", digest: \"${initrd_digest}\"}}" <<<"${image_entry}") + echo -e "${initrd_location}\n${initrd_digest}" + fi + fi + else + release_cache_key=${version}-${flavor}-${arch}-${path_suffix} + location=$( + # shellcheck disable=SC2015 + [[ -v ubuntu_image_url_release_cache[${release_cache_key}] ]] && echo "${ubuntu_image_url_release_cache[${release_cache_key}]}" || + ubuntu_image_url_release "${version}" "${flavor}" "${arch}" "${path_suffix}" + ) + ubuntu_image_url_release_cache[${release_cache_key}]="${location}" + if [[ -z ${location} ]]; then + echo "Failed to get the release image location for ${location_basename}" >&2 + continue + elif [[ ${location} == "${location_before}" ]]; then + continue + fi + image_entry="{\"location\": \"${location}\", \"arch\": \"${limayaml_arch}\"}" + echo "${location}" + if [[ ${kernel_location} != "null" ]]; then + kernel_info=$(ubuntu_kernel_info_for_image_url "${location}") + IFS=$'\t' read -r kernel_location kernel_digest initrd_location initrd_digest <<<"${kernel_info}" + if [[ -n ${kernel_location} ]]; then + image_entry=$(jq ". + {kernel: {location: \"${kernel_location}\"}}" <<<"${image_entry}") + [[ ${kernel_cmdline} != "null" ]] && image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") + echo "${kernel_location}" + fi + if [[ -n ${initrd_location} ]]; then + image_entry=$(jq ". + {initrd: {location: \"${initrd_location}\"}}" <<<"${image_entry}") + echo "${initrd_location}" + fi + fi + fi + limactl edit --log-level error --set " + [(.images.[] | path)].[${index}] as \$path| + setpath(\$path; ${image_entry}) + .images[${index}].[] style = \"double\" + " "${template}" + done +done