From fce9dea77e1ace2088c32eda1ae2d338f1624a6c Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Fri, 4 Apr 2025 10:25:41 +0200 Subject: [PATCH 1/2] Add a script to publish to generate the publish commands --- scripts/push-registry.py | 198 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100755 scripts/push-registry.py diff --git a/scripts/push-registry.py b/scripts/push-registry.py new file mode 100755 index 0000000000..9796311317 --- /dev/null +++ b/scripts/push-registry.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 + +# Can be run without dependencies with +# +# uv run --with requests,pyyaml scripts/push-registry.py + +import os +import tempfile +import requests +import json +import yaml +import sys +from logging import info, basicConfig, INFO + +yaml_db_path = "themes/default/data/registry/packages" +docs_path = "themes/default/content/registry/packages" +tmpdir = tempfile.mkdtemp() + +basicConfig(level=INFO) + +info(f"Tempdir: {tmpdir}") + +run_script = [] + +target = None +if len(sys.argv) >= 2: + target = sys.argv[1] + +publishers = { + "1Password": "1Password", + "airbytehq": "airbytehq", + "Aviatrix": "Aviatrix", + "azaurus1": "azaurus1", + "castai": "castai", + "celest-dev": "celest-dev", + "Checkly": "checkly", + "Christian Nunciato": "christian_nunciato", + "Chronosphere": "chronosphere", + "Cisco": "cisco", + "civo": "civo", + "constellix": "constellix", + "CrowdStrike": "crowdstrike", + "CTFer.io": "ctfer", + "Daniel Muehlbachler-Pietrzykowski": "daniel_muehlbachler_pietrzykowski", + "DataRobot, Inc.": "datarobot", + "Defang": "defang", + "Descope": "descope", + "dirien": "dirien", + "dmacvicar": "dmacvicar", + "Equinix": "equinix", + "EventStore": "eventstore", + "fivetran": "fivetran", + "fortinetdev": "fortinetdev", + "Genesiscloud": "genesiscloud", + "goauthentik": "goauthentik", + "Grapl Security": "grapl_security", + "hashicorp": "hashicorp", + "honeycombio": "honeycomb", + "Ian Wahbe": "ian_wahbe", + "Impart Security": "impart_security", + "incident-io": "incident_io", + "jfrog": "jfrog", + "joneshf": "joneshf", + "komminarlabs": "komminarlabs", + "kong": "kong", + "Koyeb": "koyeb", + "labd": "labd", + "lbrlabs": "lbrlabs", + "Lee Zen": "lee_zen", + "littlejo": "littlejo", + "lucky3028": "lucky3028", + "netlify": "netlify", + "Nuage-Studio": "nuage_studio", + "onelogin": "onelogin", + "oun": "oun", + "outscale": "outscale", + "OVHcloud": "ovhcloud", + "Paul Stack": "paul_stack", + "pgEdge": "pgedge", + "Piers Karsenbarg": "piers_karsenbarg", + "pinecone-io": "pinecone", + "planetscale": "planetscale", + "port-labs": "port_labs", + "prefecthq": "prefecthq", + "Prodvana": "prodvana", + "propelauth": "propelauth", + "Pulumi": "pulumi", + "pulumiverse - Marcel Arns": "pulumiverse", + "pulumiverse": "pulumiverse", + "Pulumiverse": "pulumiverse", + "qdrant": "qdrant", + "rancher": "rancher", + "RedisLabs": "redislabs", + "redpanda-data": "redpanda_data", + "Rootly": "rootly", + "Runpod": "runpod", + "splightplatform": "splightplatform", + "supabase": "supabase", + "Symbiosis": "symbiosis", + "temporalio": "temporalio", + "terraform-lxd": "terraform_lxd", + "Theo Gravity": "theo_gravity", + "Three141": "three141", + "Threefold": "threefold", + "timescale": "timescale", + "Twingate": "twingate", + "UpCloudLtd": "upcloudltd", + "Upstash": "upstash", + "vantage-sh": "vantage-sh", + "Vates": "vates", + "Volcengine": "volcengine", + "Wttech": "wttech", + "zenduty": "zenduty", + "Zscaler": "zscaler", +} + +for filename in os.listdir(yaml_db_path): + if not filename.endswith('.yaml'): + continue + if target and target != filename.removesuffix(".yaml"): + continue + + filepath = os.path.join(yaml_db_path, filename) + with open(filepath) as file: + data = yaml.safe_load(file) + + version = data.get('version') + + schema_file_path = data.get('schema_file_path') + schema_url = data.get( + 'schema_file_url', + f"https://raw.githubusercontent.com/{data.get('repo_url').removeprefix("https://github.com/")}/refs/tags/{version}/{schema_file_path}", + ) + + source = 'pulumi' if 'registry.opentofu.org' not in schema_url else "opentofu" + name = data.get('name') + publisher_display = data.get('publisher') + + + if publisher_display == "DEPRECATED": + continue + elif publisher_display not in publishers: + raise Exception(f"Missing publisher entry for {publisher_display}") + publisher = publishers[publisher_display] + + + package_version_name = f"{source}/{publisher}/{name}@{version}" + info(f"Preparing command for {package_version_name}") + + # We skip azure-native-v1, since it will (1) never be updated again and (2) isn't + # actually a package, it's an alias to the azure native package. + # + # If azure-native-v2 ships (with azure-native now at v3), we will want to exclude + # azure-native-v2 also. + if name.startswith("azure-native") and name != "azure-native": + continue + + api_url = f"https://api.pulumi.com/api/registry/preview/packages/{source}/{publisher}/{name}/versions/{version}" + existence_check = requests.head(api_url).status_code + + if existence_check == 404: + schema_file = os.path.join(tmpdir, name, "schema.json") + os.makedirs(os.path.dirname(schema_file), exist_ok=True) + + with open(schema_file, 'w') as sf: + response = requests.get(schema_url) + response.raise_for_status() + + schema = None + if (schema_file_path is not None and schema_file_path.endswith(".yaml")) or \ + (schema_url is not None and schema_url.endswith(".yaml")): + schema = yaml.safe_load(response.content) + else: + schema = json.loads(response.content) + + if "version" not in schema: + schema["version"] = version.removeprefix("v") + json.dump(schema, sf) + + cmd = [ + "pulumi", "package", "publish", schema_file, + f"--readme={docs_path}/{name}/_index.md", + f"--source={source}", + f"--publisher={publisher}", + ] + + if os.path.isfile(os.path.join(docs_path, name, "_installation-configuration.md")): + cmd.extend(["--installation-configuration", f"{docs_path}/{name}/installation-configuration.md"]) + + run_script.append(' '.join(cmd)) + elif existence_check == 200: + info("Package already exists in the registry DB") + else: + info(f"Unable to check on package {package_version_name}") + exit(1) + +for cmd in run_script: + print(cmd) From ddbb75b3b1c373456b2304a90fe4c18eefaaf7e8 Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Thu, 17 Apr 2025 19:02:13 +0200 Subject: [PATCH 2/2] Push to the live registry after publishing --- .github/workflows/pull-request.yml | 13 ++ .github/workflows/push.yml | 6 + scripts/{ => ci}/push-registry.py | 219 +++++++++++++++++++++-------- 3 files changed, 178 insertions(+), 60 deletions(-) rename scripts/{ => ci}/push-registry.py (54%) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b7417fb514..082174e46d 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -19,6 +19,7 @@ jobs: - lint-markdown - lint-scripts - preview + - test-live-publish runs-on: ubuntu-latest steps: - uses: guibranco/github-status-action-v2@0849440ec82c5fa69b2377725b9b7852a3977e76 # v1.1.13 @@ -77,6 +78,18 @@ jobs: - name: Run Linter run: yarn run lint + test-live-publish: + name: Test Live Registry Publish + runs-on: ubuntu-latest + steps: + - name: Check out branch + uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - run: uv run --with requests,pyyaml scripts/ci/push-registry.py --dry-run + env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + # Preview runs a registry build into a commit specific S3 bucket to preview changes. # # A link to the generated build is appended to the PR on each commit. diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 752943eb83..9137f5b024 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -64,6 +64,12 @@ jobs: with: name: origin-bucket-metadata path: origin-bucket-metadata.json + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Push to the Live Registry + run: uv run --with requests,pyyaml ./scripts/ci/push-registry.py + env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} notify: if: failure() diff --git a/scripts/push-registry.py b/scripts/ci/push-registry.py similarity index 54% rename from scripts/push-registry.py rename to scripts/ci/push-registry.py index 9796311317..46d5e6a619 100755 --- a/scripts/push-registry.py +++ b/scripts/ci/push-registry.py @@ -2,122 +2,203 @@ # Can be run without dependencies with # -# uv run --with requests,pyyaml scripts/push-registry.py +# uv run --with requests,pyyaml scripts/ci/push-registry.py import os import tempfile import requests +import subprocess import json import yaml -import sys -from logging import info, basicConfig, INFO +import argparse +from logging import info, warning, basicConfig, WARNING + +basicConfig(level=WARNING) yaml_db_path = "themes/default/data/registry/packages" docs_path = "themes/default/content/registry/packages" +backend_url = os.getenv("PULUMI_BACKEND_URL") or "https://api.pulumi.com/api" tmpdir = tempfile.mkdtemp() -basicConfig(level=INFO) +token = os.getenv("PULUMI_ACCESS_TOKEN") -info(f"Tempdir: {tmpdir}") +parser = argparse.ArgumentParser( + prog='push-registry', + description="""Push the contents of the pulumi/registry GitHub repo to the live registry service + + PULUMI_ACCESS_TOKEN must be set to publish. + PULUMI_BACKEND_URL may be set. Defaults to "https://api.pulumi.com/api". + """, + epilog='This is a Pulumi internal tool - it is not intended for external use') +parser.add_argument("--target", default=None, help='Only focus on one provider. This should just be the name, like "aws"') +parser.add_argument("--dry-run", action='store_true', help="Don't actually publish provider, but do everything else") -run_script = [] +args = parser.parse_args() -target = None -if len(sys.argv) >= 2: - target = sys.argv[1] +info(f"Tempdir: {tmpdir}") publishers = { "1Password": "1Password", - "airbytehq": "airbytehq", "Aviatrix": "Aviatrix", - "azaurus1": "azaurus1", - "castai": "castai", - "celest-dev": "celest-dev", + "CTFer.io": "ctfer", "Checkly": "checkly", "Christian Nunciato": "christian_nunciato", "Chronosphere": "chronosphere", "Cisco": "cisco", - "civo": "civo", - "constellix": "constellix", "CrowdStrike": "crowdstrike", - "CTFer.io": "ctfer", "Daniel Muehlbachler-Pietrzykowski": "daniel_muehlbachler_pietrzykowski", "DataRobot, Inc.": "datarobot", "Defang": "defang", "Descope": "descope", - "dirien": "dirien", - "dmacvicar": "dmacvicar", "Equinix": "equinix", "EventStore": "eventstore", + "Genesiscloud": "genesiscloud", + "Grapl Security": "grapl_security", + "Ian Wahbe": "ian_wahbe", + "Impart Security": "impart_security", + "Koyeb": "koyeb", + "Lee Zen": "lee_zen", + "Nuage-Studio": "nuage_studio", + "OVHcloud": "ovhcloud", + "Paul Stack": "paul_stack", + "Piers Karsenbarg": "piers_karsenbarg", + "Prodvana": "prodvana", + "Pulumi": "pulumi", + "Pulumiverse": "pulumiverse", + "RedisLabs": "redislabs", + "Rootly": "rootly", + "Runpod": "runpod", + "Symbiosis": "symbiosis", + "Theo Gravity": "theo_gravity", + "Three141": "three141", + "Threefold": "threefold", + "Twingate": "twingate", + "UpCloudLtd": "upcloudltd", + "Upstash": "upstash", + "Vates": "vates", + "Volcengine": "volcengine", + "Wttech": "wttech", + "Zscaler": "zscaler", + "airbytehq": "airbytehq", + "akeyless-community": "akeyless-community", + "aptible": "aptible", + "athenz": "athenz", + "azaurus1": "azaurus1", + "castai": "castai", + "celest-dev": "celest-dev", + "chainguard-dev": "chainguard-dev", + "checkpointsw": "checkpointsw", + "ciscodevnet": "ciscodevnet", + "ciscodevnet": "ciscodevnet", + "civo": "civo", + "cloudfoundry-community": "cloudfoundry-community", + "coder": "coder", + "constellix": "constellix", + "coralogix": "coralogix", + "cox-automotive": "cox-automotive", + "cyralinc": "cyralinc", + "datadrivers": "datadrivers", + "dell": "dell", + "dell": "dell", + "dell": "dell", + "denouche": "denouche", + "dirien": "dirien", + "dmacvicar": "dmacvicar", + "dome9": "dome9", + "drfaust92": "drfaust92", + "e-breuninger": "e-breuninger", + "edge-center": "edge-center", + "elastic": "elastic", + "elastic-infra": "elastic-infra", + "ferlab-ste-justine": "ferlab-ste-justine", + "ferlab-ste-justine": "ferlab-ste-justine", "fivetran": "fivetran", + "flexibleenginecloud": "flexibleenginecloud", "fortinetdev": "fortinetdev", - "Genesiscloud": "genesiscloud", + "g-core": "g-core", + "glesys": "glesys", "goauthentik": "goauthentik", - "Grapl Security": "grapl_security", "hashicorp": "hashicorp", "honeycombio": "honeycomb", - "Ian Wahbe": "ian_wahbe", - "Impart Security": "impart_security", + "hpe": "hpe", + "ibm-cloud": "ibm-cloud", + "imperva": "imperva", "incident-io": "incident_io", + "infobloxopen": "infobloxopen", + "ionos-cloud": "ionos-cloud", + "iterative": "iterative", + "jdamata": "jdamata", "jfrog": "jfrog", "joneshf": "joneshf", + "juju": "juju", + "k-yomo": "k-yomo", "komminarlabs": "komminarlabs", "kong": "kong", - "Koyeb": "koyeb", "labd": "labd", + "lacework": "lacework", "lbrlabs": "lbrlabs", - "Lee Zen": "lee_zen", "littlejo": "littlejo", + "logdna": "logdna", + "logzio": "logzio", "lucky3028": "lucky3028", + "mastercard": "mastercard", + "maxlaverse": "maxlaverse", + "megaport": "megaport", + "mrolla": "mrolla", + "nats-io": "nats-io", + "netapp": "netapp", "netlify": "netlify", - "Nuage-Studio": "nuage_studio", + "octopusdeploylabs": "octopusdeploylabs", "onelogin": "onelogin", + "opennebula": "opennebula", + "opensearch-project": "opensearch-project", + "opentelekomcloud": "opentelekomcloud", "oun": "oun", "outscale": "outscale", - "OVHcloud": "ovhcloud", - "Paul Stack": "paul_stack", + "paloaltonetworks": "paloaltonetworks", + "paloaltonetworks": "paloaltonetworks", + "pan-net": "pan-net", + "paultyng": "paultyng", "pgEdge": "pgedge", - "Piers Karsenbarg": "piers_karsenbarg", + "philips-software": "philips-software", "pinecone-io": "pinecone", "planetscale": "planetscale", "port-labs": "port_labs", "prefecthq": "prefecthq", - "Prodvana": "prodvana", "propelauth": "propelauth", - "Pulumi": "pulumi", "pulumiverse - Marcel Arns": "pulumiverse", "pulumiverse": "pulumiverse", - "Pulumiverse": "pulumiverse", "qdrant": "qdrant", "rancher": "rancher", - "RedisLabs": "redislabs", "redpanda-data": "redpanda_data", - "Rootly": "rootly", - "Runpod": "runpod", + "rollbar": "rollbar", + "selectel": "selectel", + "spectrocloud": "spectrocloud", "splightplatform": "splightplatform", "supabase": "supabase", - "Symbiosis": "symbiosis", + "sysdiglabs": "sysdiglabs", "temporalio": "temporalio", + "tencentcloudstack": "tencentcloudstack", "terraform-lxd": "terraform_lxd", - "Theo Gravity": "theo_gravity", - "Three141": "three141", - "Threefold": "threefold", + "terraform-routeros": "terraform-routeros", "timescale": "timescale", - "Twingate": "twingate", - "UpCloudLtd": "upcloudltd", - "Upstash": "upstash", + "ucloud": "ucloud", "vantage-sh": "vantage-sh", - "Vates": "vates", - "Volcengine": "volcengine", - "Wttech": "wttech", + "vk-cs": "vk-cs", + "vmware": "vmware", + "vmware": "vmware", + "vmware": "vmware", + "vmware": "vmware", + "vmware": "vmware", "zenduty": "zenduty", - "Zscaler": "zscaler", } +has_failed = False + for filename in os.listdir(yaml_db_path): if not filename.endswith('.yaml'): continue - if target and target != filename.removesuffix(".yaml"): + if args.target and args.target != filename.removesuffix(".yaml"): continue filepath = os.path.join(yaml_db_path, filename) @@ -140,11 +221,11 @@ if publisher_display == "DEPRECATED": continue elif publisher_display not in publishers: - raise Exception(f"Missing publisher entry for {publisher_display}") + raise Exception(f'Missing publisher entry for "{publisher_display}"') publisher = publishers[publisher_display] - package_version_name = f"{source}/{publisher}/{name}@{version}" + package_version_name = f"{source}/{publisher}/{name}@{version.removeprefix("v")}" info(f"Preparing command for {package_version_name}") # We skip azure-native-v1, since it will (1) never be updated again and (2) isn't @@ -155,8 +236,11 @@ if name.startswith("azure-native") and name != "azure-native": continue - api_url = f"https://api.pulumi.com/api/registry/preview/packages/{source}/{publisher}/{name}/versions/{version}" - existence_check = requests.head(api_url).status_code + + api_url = f"{backend_url}/preview/registry/packages/{source}/{publisher}/{name}/versions/{version.removeprefix("v")}" + existence_check = requests.get(api_url, headers={ + "Authorization": f"token {token}", + }).status_code if existence_check == 404: schema_file = os.path.join(tmpdir, name, "schema.json") @@ -173,26 +257,41 @@ else: schema = json.loads(response.content) - if "version" not in schema: + + # We *correct* the version if it isn't present *or* if it disagrees with the + # version that has been published in github.com/pulumi/registry. + if "version" not in schema or schema["version"] != version.removeprefix("v"): schema["version"] = version.removeprefix("v") json.dump(schema, sf) + cmd = [ - "pulumi", "package", "publish", schema_file, - f"--readme={docs_path}/{name}/_index.md", - f"--source={source}", - f"--publisher={publisher}", + "pulumi", "package", "publish", + schema_file, + "--readme", f"{docs_path}/{name}/_index.md", + "--source", source, + "--publisher", publisher, ] if os.path.isfile(os.path.join(docs_path, name, "_installation-configuration.md")): cmd.extend(["--installation-configuration", f"{docs_path}/{name}/installation-configuration.md"]) - run_script.append(' '.join(cmd)) + if args.dry_run: + print(" ".join(cmd)) + else: + try: + subprocess.run(cmd, shell=False, check=True) + except subprocess.CalledProcessError: + warning(f"Failed to run {" ".join(cmd)}") + has_failed=True + + os.remove(schema_file) + os.removedirs(os.path.dirname(schema_file)) elif existence_check == 200: info("Package already exists in the registry DB") else: - info(f"Unable to check on package {package_version_name}") - exit(1) + warning(f"Unable to check on package {package_version_name}") + has_failed=True -for cmd in run_script: - print(cmd) +if has_failed: + exit(1)