From 941e1e0e5b235dfb9b158d917d31a63a8e48647b Mon Sep 17 00:00:00 2001
From: James Healy <james@buildkite.com>
Date: Sun, 10 Nov 2024 18:26:52 +1100
Subject: [PATCH 01/15] Update rails-ci pipeline to run on Buildkite hosted
 agents

An experiment in changing the rails CI pipeline from "self-hosted"
agents to "hosted" agents, a recently release Buildkite feature [1].

The hosted agents linux environment is superficially quite similar to
the Elastic Stack for AWS, so the required changes are fairly minimal.
Roughly half the changes are to take advantage of some performance
optimisations available on hosted agents (like cache volumes, and
remote buildkit builders with cache that last across builds).

The essential changes:

* Read the OCI registry from the environment rather than hard code an
  ECR registry. The current self-hosted agents run in AWS and can access
  ECR, but the hosted agent environment has access to its own registry
  specifically for use cases like this - building an image at the start
  of the build and then reusing it in later jobs
* Changing the queue from `default` or `builder`, to `hosted`

Optimisations:

* There's no need to use the docker-compose plugins cache_from and
  image_name shenanigans. The images built at the start of each build
  use a remote buildkit builder with cache that is s hared between
  builds. The cache is typically warm, and when it is the image build
  time drops from ~2 mins to ~18sec
* Use plain buildkit to build the images, without the docker compose
  plugin. This avoids the image being exported from buildkit to docker,
  and when the buildkit cache is warm the jobs complete in as little as
  18s. This bypasses the docker-compse built in support for separating
  building and running, but the docker-compose.yml already kinda
  bypasses that by hard coding the image used in the run jobs (using the
  IMAGE_NAME env var)
* ~Create a cache volume for ruby gems that are installed in docker
  during the initial step. This shaves ~30s off the build time~

[1] https://buildkite.com/docs/pipelines/hosted-agents/overview
---
 lib/buildkite/config/build_context.rb |  2 +-
 lib/buildkite/config/docker_build.rb  | 17 ++---------------
 lib/buildkite/config/rake_command.rb  |  3 +--
 pipelines/rails-ci/initial.yml        | 10 ++++++++--
 pipelines/rails-ci/pipeline.rb        |  2 +-
 5 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb
index c5f77603..9f6874cf 100644
--- a/lib/buildkite/config/build_context.rb
+++ b/lib/buildkite/config/build_context.rb
@@ -190,7 +190,7 @@ def min_ruby
       end
 
       def remote_image_base
-        "973266071021.dkr.ecr.us-east-1.amazonaws.com/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+        ENV.fetch("REGISTRY") + "/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
       end
   end
 end
diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb
index b43f6301..ff64c83d 100644
--- a/lib/buildkite/config/docker_build.rb
+++ b/lib/buildkite/config/docker_build.rb
@@ -30,11 +30,7 @@ def cache_from(build_context)
       end
 
       def build_push(build_context)
-        [
-          build_context.local_branch =~ /:/ ?
-            build_context.image_name_for("pr-#{build_context.pull_request}") :
-            build_context.image_name_for("br-#{build_context.local_branch}"),
-        ]
+        build_context.image_name_for(build_context.build_id, prefix: nil)
       end
     end
 
@@ -66,20 +62,11 @@ def builder(ruby)
             compressed: ".buildkite.tgz"
           }
 
-          plugin :docker_compose, {
-            build: "base",
-            config: ".buildkite/docker-compose.yml",
-            env: %w[PRE_STEPS RACK],
-            "image-name" => build_context.ruby.image_name_for(build_context.build_id),
-            "cache-from" => cache_from(build_context),
-            push: build_push(build_context),
-            "image-repository" => build_context.image_base,
-          }
+          command "docker build --push --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image} --tag #{build_push(build_context)} --file .buildkite/Dockerfile ."
 
           env({
             BUNDLER: build_context.bundler,
             RUBYGEMS: build_context.rubygems,
-            RUBY_IMAGE: build_context.ruby.ruby_image,
             encrypted_0fb9444d0374_key: nil,
             encrypted_0fb9444d0374_iv: nil
           })
diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index 7c78a9f8..1b33996b 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -52,8 +52,7 @@ def install_plugins(service = "default", env = nil, dir = ".")
         plugin :docker_compose, {
           "env" => env,
           "run" => service,
-          "pull" => service,
-          "pull-retries" => 3,
+          "tty" => "true",
           "config" => ".buildkite/docker-compose.yml",
           "shell" => ["runner", *dir],
         }.compact
diff --git a/pipelines/rails-ci/initial.yml b/pipelines/rails-ci/initial.yml
index 299245db..2dc06638 100644
--- a/pipelines/rails-ci/initial.yml
+++ b/pipelines/rails-ci/initial.yml
@@ -4,6 +4,9 @@
 steps:
   - name: ":pipeline: rails-initial-pipeline"
     command: |
+      echo "Fetching registry details"
+      export REGISTRY="$$(nsc workspace describe -o json -k registry_url)"
+
       PATH=/bin:/usr/bin
       set -e
 
@@ -30,17 +33,18 @@ steps:
       echo "Fetching pull-request metadata:"
       (docker run --rm \
         -v "$$PWD":/app:ro -w /app \
+        -v "$$PWD/cache/bundler":/usr/local/bundle \
         -e GITHUB_PUBLIC_REPO_TOKEN \
         -e BUILDKITE_REPO \
         -e BUILDKITE_PULL_REQUEST \
         ruby:latest \
         .buildkite/bin/fetch-pr > .buildkite/tmp/.pr-meta.json) || true
 
+
       echo "Generating pipeline:"
       sh -c "$$PIPELINE_COMMAND"
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
-
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -58,6 +62,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e CI
         -e BUILDKITE
         -e BUILDKITE_AGENT_META_DATA_QUEUE
@@ -72,9 +77,10 @@ steps:
         -e DOCKER_IMAGE
         -e RUN_QUEUE
         -e QUEUE
+        -e REGISTRY
         ruby:latest
         .buildkite/bin/pipeline-generate rails-ci |
         buildkite-agent pipeline upload
     timeout_in_minutes: 5
     agents:
-      queue: "${QUEUE-builder}"
+      queue: hosted
diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb
index fe597332..b5e74a9d 100644
--- a/pipelines/rails-ci/pipeline.rb
+++ b/pipelines/rails-ci/pipeline.rb
@@ -7,7 +7,7 @@
   use Buildkite::Config::RakeCommand
   use Buildkite::Config::RubyGroup
 
-  plugin :docker_compose, "docker-compose#v4.16.0"
+  plugin :docker_compose, "docker-compose#v5.4.1"
   plugin :artifacts, "artifacts#v1.9.3"
 
   if build_context.nightly?

From 147a9447f5096fe859d935077ae1666ca3ca28f9 Mon Sep 17 00:00:00 2001
From: zzak <zzakscott@gmail.com>
Date: Fri, 14 Feb 2025 14:52:46 +0900
Subject: [PATCH 02/15] Use cluster-secrets for mainline and docs-preview
 deploys

---
 lib/buildkite/config/rake_command.rb | 12 +++++++++---
 pipelines/docs-preview/pipeline.rb   |  7 +++++++
 pipelines/rails-ci/pipeline.rb       |  1 +
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index 1b33996b..0abf79a9 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -33,7 +33,7 @@ def build_env(build_context, pre_steps, env)
         env
       end
 
-      def install_plugins(service = "default", env = nil, dir = ".")
+      def install_plugins(service = "default", env = nil, dir = ".", mainline: false)
         plugin :artifacts, {
             download: ".dockerignore"
           }
@@ -49,6 +49,12 @@ def install_plugins(service = "default", env = nil, dir = ".")
           compressed: ".buildkite.tgz"
         }
 
+        if mainline
+          plugin :secrets, {
+            env: "main_env"
+          }
+        end
+
         plugin :docker_compose, {
           "env" => env,
           "run" => service,
@@ -72,7 +78,7 @@ def bundle(command, label:, env: nil)
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command command
 
-          install_plugins
+          install_plugins(mainline: build_context.mainline)
 
           env build_env(build_context, nil, env)
 
@@ -98,7 +104,7 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env:
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command "rake #{task}"
 
-          install_plugins(service,  %w[PRE_STEPS RACK], dir)
+          install_plugins(service,  %w[PRE_STEPS RACK], dir, mainline: build_context.mainline)
 
           env build_env(build_context, pre_steps, env)
 
diff --git a/pipelines/docs-preview/pipeline.rb b/pipelines/docs-preview/pipeline.rb
index b18c3586..88c7617e 100644
--- a/pipelines/docs-preview/pipeline.rb
+++ b/pipelines/docs-preview/pipeline.rb
@@ -6,6 +6,7 @@
 
   plugin :docker, "docker#v5.10.0"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   build_context = context.extensions.find(Buildkite::Config::BuildContext)
   build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.3"))
@@ -50,6 +51,9 @@
     key "deploy"
     depends_on "build"
     timeout_in_minutes 15
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     plugin :docker, {
       environment: [
         "BUILDKITE_BRANCH",
@@ -83,6 +87,9 @@
       download: ".buildkite/bin/docs-preview-annotate",
       compressed: ".buildkite.tgz"
     }
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     command "sh -c \"$$ANNOTATE_COMMAND\" | buildkite-agent annotate --style info"
     # CLOUDFLARE_API_TOKEN is used to fetch preview URL from latest deployment
     env "ANNOTATE_COMMAND" => <<~ANNOTATE.gsub(/[[:space:]]+/, " ").strip
diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb
index b5e74a9d..3e1c03bf 100644
--- a/pipelines/rails-ci/pipeline.rb
+++ b/pipelines/rails-ci/pipeline.rb
@@ -9,6 +9,7 @@
 
   plugin :docker_compose, "docker-compose#v5.4.1"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   if build_context.nightly?
     build_context.rubies << Buildkite::Config::RubyConfig.master_ruby

From 1d8640ab65d768594e16b44abe23909783c47ba9 Mon Sep 17 00:00:00 2001
From: zzak <zzakscott@gmail.com>
Date: Fri, 14 Feb 2025 14:57:51 +0900
Subject: [PATCH 03/15] Always build image for docs-preview steps

---
 pipelines/docs-preview/pipeline.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pipelines/docs-preview/pipeline.rb b/pipelines/docs-preview/pipeline.rb
index 88c7617e..59b2d234 100644
--- a/pipelines/docs-preview/pipeline.rb
+++ b/pipelines/docs-preview/pipeline.rb
@@ -3,6 +3,7 @@
 Buildkite::Builder.pipeline do
   require "buildkite_config"
   use Buildkite::Config::BuildContext
+  use Buildkite::Config::DockerBuild
 
   plugin :docker, "docker#v5.10.0"
   plugin :artifacts, "artifacts#v1.9.3"
@@ -23,13 +24,16 @@
     next
   end
 
+  builder build_context.ruby
+
   command do
     label "build", emoji: :rails
+    depends_on "docker-image-#{build_context.ruby.image_key}"
     key "build"
     command "bundle install && bundle exec rake preview_docs"
     timeout_in_minutes 15
     plugin :docker, {
-      image: build_context.image_name_for("br-main", prefix: nil),
+      image: build_context.image_name_for(build_context.build_id, prefix: nil),
       environment: [
         "BUILDKITE_BRANCH",
         "BUILDKITE_BUILD_CREATOR",

From f2b1e85bae647ad0e41aa69b7d743d2e9e4bfc2c Mon Sep 17 00:00:00 2001
From: zzak <zzakscott@gmail.com>
Date: Fri, 14 Feb 2025 14:59:36 +0900
Subject: [PATCH 04/15] [docs-preview] Prefer alias url when available

---
 bin/docs-preview-annotate | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bin/docs-preview-annotate b/bin/docs-preview-annotate
index c24a3be0..343b4417 100755
--- a/bin/docs-preview-annotate
+++ b/bin/docs-preview-annotate
@@ -27,12 +27,13 @@ end
 
 json = JSON.parse(response.body)
 result = json["result"].first
+url = result["aliases"]&.first || result["url"]
 
 plan = <<~PLAN
 #### :writing_hand: rails/docs-preview:
 
-* <a href="#{result["url"]}/api">:link: API</a>
-* <a href="#{result["url"]}/guides">:link: Guides</a>
+* <a href="#{url}/api">:link: API</a>
+* <a href="#{url}/guides">:link: Guides</a>
 PLAN
 
 puts plan

From 67c0847ff9bd008e5389773120e03008ef64bf30 Mon Sep 17 00:00:00 2001
From: zzak <zzakscott@gmail.com>
Date: Fri, 14 Feb 2025 16:08:14 +0900
Subject: [PATCH 05/15] Make registry optional and fix tests

---
 lib/buildkite/config/build_context.rb      | 10 +++++-
 test/buildkite_config/test_docker_build.rb | 41 +++++++++-------------
 test/buildkite_config/test_rake_command.rb |  7 ++--
 3 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb
index 9f6874cf..014c09b5 100644
--- a/lib/buildkite/config/build_context.rb
+++ b/lib/buildkite/config/build_context.rb
@@ -189,8 +189,16 @@ def min_ruby
         Gem::Version.new($1 || "2.0")
       end
 
+      def registry
+        ENV["REGISTRY"] || "973266071021.dkr.ecr.us-east-1.amazonaws.com"
+      end
+
+      def image_name
+        "#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+      end
+
       def remote_image_base
-        ENV.fetch("REGISTRY") + "/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+        [registry, image_name].join("/")
       end
   end
 end
diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb
index e97dde3c..4d2ef6c9 100644
--- a/test/buildkite_config/test_docker_build.rb
+++ b/test/buildkite_config/test_docker_build.rb
@@ -19,7 +19,6 @@ def test_builder_with_ruby_config_using_string_version
 
     assert_equal ":docker: builder:3.2", pipeline.to_h["steps"][0]["label"]
     assert_equal "docker-image-builder-3-2", pipeline.to_h["steps"][0]["key"]
-    assert_equal "builder:3.2", pipeline.to_h["steps"][0]["env"]["RUBY_IMAGE"]
   end
 
   def test_builder_artifacts
@@ -60,25 +59,16 @@ def test_builder_compose_plugin
       end
     end
 
-    plugins = pipeline.to_h["steps"][0]["plugins"]
-
-    compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+    command = pipeline.to_h["steps"][0]["command"].first
 
-    %w[image-name cache-from push build config env image-repository].each do |key|
-      assert_includes compose, key
-    end
+    expected = <<~COMMAND.squish
+      docker build --push
+        --build-arg RUBY_IMAGE=3.2
+        --tag buildkite-config-base:3-2-local
+        --file .buildkite/Dockerfile .
+    COMMAND
 
-    assert_equal "3-2-local", compose["image-name"]
-    assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"]
-
-    assert_equal "base", compose["build"]
-    assert_equal ".buildkite/docker-compose.yml", compose["config"]
-    assert_includes compose["env"], "PRE_STEPS"
-    assert_includes compose["env"], "RACK"
-    assert_equal "buildkite-config-base", compose["image-repository"]
+    assert_equal expected.strip, command
   end
 
   def test_builder_timeout_default
@@ -143,14 +133,15 @@ def test_builder_gem_version
       end
     end
 
-    plugins = pipeline.to_h["steps"][0]["plugins"]
+    command = pipeline.to_h["steps"][0]["command"].first
 
-    compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+    expected = <<~COMMAND.squish
+      docker build --push
+        --build-arg RUBY_IMAGE=ruby:1.9.3
+        --tag buildkite-config-base:ruby-1-9-3-local
+        --file .buildkite/Dockerfile .
+    COMMAND
 
-    assert_equal "ruby-1-9-3-local", compose["image-name"]
-    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"]
+    assert_equal expected.strip, command
   end
 end
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index 77924533..0e72a909 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -207,7 +207,7 @@ def test_compose
       plugin.key?("docker-compose#v1.0")
     }.fetch("docker-compose#v1.0")
 
-    %w[env run pull config shell].each do |key|
+    %w[env run config shell].each do |key|
       assert_includes compose, key
     end
 
@@ -215,7 +215,6 @@ def test_compose
     assert_includes compose["env"], "RACK"
 
     assert_equal "default", compose["run"]
-    assert_equal "default", compose["pull"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]
   end
@@ -263,12 +262,11 @@ def test_docker_compose_plugin_service
       plugin.key?("docker-compose#v1.0")
     }.fetch("docker-compose#v1.0")
 
-    %w[run pull].each do |key|
+    %w[run].each do |key|
       assert_includes compose, key
     end
 
     assert_equal "myservice", compose["run"]
-    assert_equal "myservice", compose["pull"]
   end
 
   def test_env_yjit
@@ -453,7 +451,6 @@ def test_bundle_command
 
     assert_not_includes compose, "env"
     assert_equal "default", compose["run"]
-    assert_equal "default", compose["pull"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "."], compose["shell"]
 

From ad7ebb0b0676723bd2c5293575fe2ba965bba7cf Mon Sep 17 00:00:00 2001
From: zzak <zzakscott@gmail.com>
Date: Fri, 14 Feb 2025 16:10:37 +0900
Subject: [PATCH 06/15] Pause setting queue to hosted until after
 buildkit+secrets

---
 pipelines/rails-ci/initial.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pipelines/rails-ci/initial.yml b/pipelines/rails-ci/initial.yml
index 2dc06638..2453f551 100644
--- a/pipelines/rails-ci/initial.yml
+++ b/pipelines/rails-ci/initial.yml
@@ -40,7 +40,6 @@ steps:
         ruby:latest \
         .buildkite/bin/fetch-pr > .buildkite/tmp/.pr-meta.json) || true
 
-
       echo "Generating pipeline:"
       sh -c "$$PIPELINE_COMMAND"
 
@@ -83,4 +82,4 @@ steps:
         buildkite-agent pipeline upload
     timeout_in_minutes: 5
     agents:
-      queue: hosted
+      queue: "${QUEUE-builder}"

From fb1aad23a592d286225e2fddac76d39731665386 Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 12:11:21 +0900
Subject: [PATCH 07/15] Allow switching buildkite based on compute type

https://buildkite.com/docs/pipelines/configure/environment-variables#BUILDKITE_COMPUTE_TYPE
---
 lib/buildkite/config/build_context.rb       |  18 +++-
 lib/buildkite/config/docker_build.rb        |  39 +++++++-
 lib/buildkite/config/rake_command.rb        |  22 +++--
 test/buildkite_config/test_build_context.rb |  82 ++++++++++++++++
 test/buildkite_config/test_docker_build.rb  | 102 +++++++++++++++++++-
 test/buildkite_config/test_rake_command.rb  |  82 ++++++++++++----
 test/test_helper.rb                         |  16 ++-
 7 files changed, 321 insertions(+), 40 deletions(-)

diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb
index 014c09b5..0caf78f8 100644
--- a/lib/buildkite/config/build_context.rb
+++ b/lib/buildkite/config/build_context.rb
@@ -117,6 +117,18 @@ def pull_request
       ([ENV["BUILDKITE_PULL_REQUEST"]] - ["false"]).first
     end
 
+    def compute_type
+      ENV["BUILDKITE_COMPUTE_TYPE"] || "self-hosted"
+    end
+
+    def self_hosted?
+      compute_type == "self-hosted"
+    end
+
+    def hosted?
+      !self_hosted?
+    end
+
     def standard_queues
       [nil, "default", "builder"]
     end
@@ -190,7 +202,11 @@ def min_ruby
       end
 
       def registry
-        ENV["REGISTRY"] || "973266071021.dkr.ecr.us-east-1.amazonaws.com"
+        if hosted?
+          ENV["REGISTRY"]
+        else
+          "973266071021.dkr.ecr.us-east-1.amazonaws.com"
+        end
       end
 
       def image_name
diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb
index ff64c83d..a70486d6 100644
--- a/lib/buildkite/config/docker_build.rb
+++ b/lib/buildkite/config/docker_build.rb
@@ -30,7 +30,15 @@ def cache_from(build_context)
       end
 
       def build_push(build_context)
-        build_context.image_name_for(build_context.build_id, prefix: nil)
+        if build_context.hosted?
+          build_context.image_name_for(build_context.build_id, prefix: nil)
+        else
+          [
+            build_context.local_branch =~ /:/ ?
+              build_context.image_name_for("pr-#{build_context.pull_request}") :
+              build_context.image_name_for("br-#{build_context.local_branch}"),
+          ]
+        end
       end
     end
 
@@ -62,14 +70,37 @@ def builder(ruby)
             compressed: ".buildkite.tgz"
           }
 
-          command "docker build --push --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image} --tag #{build_push(build_context)} --file .buildkite/Dockerfile ."
+          if build_context.hosted?
+            command <<~COMMAND.squish
+              docker build --push
+                --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image}
+                --tag #{build_push(build_context)}
+                --file .buildkite/Dockerfile .
+            COMMAND
+          else
+            plugin :docker_compose, {
+              build: "base",
+              config: ".buildkite/docker-compose.yml",
+              env: %w[PRE_STEPS RACK],
+              "image-name" => build_context.ruby.image_name_for(build_context.build_id),
+              "cache-from" => cache_from(build_context),
+              push: build_push(build_context),
+              "image-repository" => build_context.image_base,
+            }
+          end
 
-          env({
+          env_opts = {
             BUNDLER: build_context.bundler,
             RUBYGEMS: build_context.rubygems,
             encrypted_0fb9444d0374_key: nil,
             encrypted_0fb9444d0374_iv: nil
-          })
+          }
+
+          if build_context.self_hosted?
+            env_opts[:RUBY_IMAGE] = build_context.ruby.ruby_image
+          end
+
+          env(env_opts)
 
           timeout_in_minutes 15
 
diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index 0abf79a9..2be65add 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -33,7 +33,7 @@ def build_env(build_context, pre_steps, env)
         env
       end
 
-      def install_plugins(service = "default", env = nil, dir = ".", mainline: false)
+      def install_plugins(service = "default", env = nil, dir = ".", build_context:)
         plugin :artifacts, {
             download: ".dockerignore"
           }
@@ -49,19 +49,27 @@ def install_plugins(service = "default", env = nil, dir = ".", mainline: false)
           compressed: ".buildkite.tgz"
         }
 
-        if mainline
+        if build_context.mainline
           plugin :secrets, {
             env: "main_env"
           }
         end
 
-        plugin :docker_compose, {
+        compose_opts = {
           "env" => env,
           "run" => service,
-          "tty" => "true",
           "config" => ".buildkite/docker-compose.yml",
           "shell" => ["runner", *dir],
-        }.compact
+        }
+
+        if build_context.hosted?
+          compose_opts["tty"] = "true"
+        else
+          compose_opts["pull"] = service
+          compose_opts["pull-retries"] = 3
+        end
+
+        plugin :docker_compose, compose_opts.compact
       end
     end
 
@@ -78,7 +86,7 @@ def bundle(command, label:, env: nil)
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command command
 
-          install_plugins(mainline: build_context.mainline)
+          install_plugins(build_context: build_context)
 
           env build_env(build_context, nil, env)
 
@@ -104,7 +112,7 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env:
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command "rake #{task}"
 
-          install_plugins(service,  %w[PRE_STEPS RACK], dir, mainline: build_context.mainline)
+          install_plugins(service,  %w[PRE_STEPS RACK], dir, build_context: build_context)
 
           env build_env(build_context, pre_steps, env)
 
diff --git a/test/buildkite_config/test_build_context.rb b/test/buildkite_config/test_build_context.rb
index 9eb3e4b0..5c28c8dd 100644
--- a/test/buildkite_config/test_build_context.rb
+++ b/test/buildkite_config/test_build_context.rb
@@ -33,6 +33,36 @@ def test_ci_env_ci
     ENV["CI"] = @before_env_ci
   end
 
+  def test_ci_env_compute_type_nil
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV.delete("BUILDKITE_COMPUTE_TYPE")
+
+    sub = create_build_context
+    assert_predicate sub, :self_hosted?
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_ci_env_compute_type_self_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
+
+    sub = create_build_context
+    assert_predicate sub, :self_hosted?
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_ci_env_compute_type_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
+
+    sub = create_build_context
+    assert_predicate sub, :hosted?
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
   def test_nightly
     @before_env_nightly = ENV["RAILS_CI_NIGHTLY"]
     ENV["RAILS_CI_NIGHTLY"] = "true"
@@ -218,6 +248,58 @@ def test_remote_image_base
     assert_equal "973266071021.dkr.ecr.us-east-1.amazonaws.com/builds", sub.send(:remote_image_base)
   end
 
+  def test_registry_default
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV.delete("BUILDKITE_COMPUTE_TYPE")
+    @before_env_registry = ENV["REGISTRY"]
+    ENV.delete("REGISTRY")
+
+    sub = create_build_context
+    assert_equal "973266071021.dkr.ecr.us-east-1.amazonaws.com", sub.send(:registry)
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+    ENV["REGISTRY"] = @before_env_registry
+  end
+
+  def test_registry_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
+    @before_env_registry = ENV["REGISTRY"]
+    ENV["REGISTRY"] = "test_registry_hosted"
+
+    sub = create_build_context
+    assert_equal "test_registry_hosted", sub.send(:registry)
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+    ENV["REGISTRY"] = @before_env_registry
+  end
+
+  def test_registry_self_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
+    @before_env_registry = ENV["REGISTRY"]
+    ENV["REGISTRY"] = "test_registry_self_hosted"
+
+    sub = create_build_context
+    assert_equal "973266071021.dkr.ecr.us-east-1.amazonaws.com", sub.send(:registry)
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+    ENV["REGISTRY"] = @before_env_registry
+  end
+
+  def test_registry_env
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV.delete("BUILDKITE_COMPUTE_TYPE")
+    @before_env_registry = ENV["REGISTRY"]
+    ENV["REGISTRY"] = "test_registry_env"
+
+    sub = create_build_context
+    assert_equal "973266071021.dkr.ecr.us-east-1.amazonaws.com", sub.send(:registry)
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+    ENV["REGISTRY"] = @before_env_registry
+  end
+
   def test_remote_image_base_standard_queues
     sub = create_build_context
 
diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb
index 4d2ef6c9..3b5372a0 100644
--- a/test/buildkite_config/test_docker_build.rb
+++ b/test/buildkite_config/test_docker_build.rb
@@ -5,6 +5,8 @@
 
 class TestDockerBuild < TestCase
   def test_builder_with_ruby_config_using_string_version
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
     pipeline = PipelineFixture.new do
       use Buildkite::Config::DockerBuild
 
@@ -19,6 +21,31 @@ def test_builder_with_ruby_config_using_string_version
 
     assert_equal ":docker: builder:3.2", pipeline.to_h["steps"][0]["label"]
     assert_equal "docker-image-builder-3-2", pipeline.to_h["steps"][0]["key"]
+    assert_equal "builder:3.2", pipeline.to_h["steps"][0]["env"]["RUBY_IMAGE"]
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_builder_with_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
+    pipeline = PipelineFixture.new do
+      use Buildkite::Config::DockerBuild
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        builder Buildkite::Config::RubyConfig.new(prefix: "builder:", version: "3.2")
+      end
+    end
+
+    %w[label key env].each do |key|
+      assert_includes pipeline.to_h["steps"][0], key
+    end
+
+    assert_equal ":docker: builder:3.2", pipeline.to_h["steps"][0]["label"]
+    assert_equal "docker-image-builder-3-2", pipeline.to_h["steps"][0]["key"]
+    assert_not_includes pipeline.to_h["steps"][0]["env"], "RUBY_IMAGE"
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end
 
   def test_builder_artifacts
@@ -34,11 +61,11 @@ def test_builder_artifacts
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     artifacts = plugins.select { |plugin|
-      plugin.key?("artifacts#v1.0")
+      plugin.key?(plugins_map[:artifacts])
     }
-    assert_equal ".dockerignore", artifacts[0]["artifacts#v1.0"]["download"]
+    assert_equal ".dockerignore", artifacts[0][plugins_map[:artifacts]]["download"]
 
-    download = artifacts[1]["artifacts#v1.0"]
+    download = artifacts[1][plugins_map[:artifacts]]
     assert_equal %w[
       .buildkite/.empty
       .buildkite/docker-compose.yml
@@ -50,7 +77,42 @@ def test_builder_artifacts
     assert_equal ".buildkite.tgz", download["compressed"]
   end
 
-  def test_builder_compose_plugin
+  def test_builder_compose_plugin_self_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
+    pipeline = PipelineFixture.new do
+      use Buildkite::Config::DockerBuild
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        builder Buildkite::Config::RubyConfig.new(version: "3.2")
+      end
+    end
+
+    plugins = pipeline.to_h["steps"][0]["plugins"]
+
+    compose = plugins.find { |plugin|
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
+
+    %w[image-name cache-from push build config env image-repository].each do |key|
+      assert_includes compose, key
+    end
+    assert_equal "3-2-local", compose["image-name"]
+    assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"]
+    assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"]
+
+    assert_equal "base", compose["build"]
+    assert_equal ".buildkite/docker-compose.yml", compose["config"]
+    assert_includes compose["env"], "PRE_STEPS"
+    assert_includes compose["env"], "RACK"
+    assert_equal "buildkite-config-base", compose["image-repository"]
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_builder_compose_plugin_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
     pipeline = PipelineFixture.new do
       use Buildkite::Config::DockerBuild
 
@@ -69,6 +131,8 @@ def test_builder_compose_plugin
     COMMAND
 
     assert_equal expected.strip, command
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end
 
   def test_builder_timeout_default
@@ -124,7 +188,9 @@ def test_builder_skip
     assert_equal({}, pipeline.to_h)
   end
 
-  def test_builder_gem_version
+  def test_builder_gem_version_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
     pipeline = PipelineFixture.new do
       use Buildkite::Config::DockerBuild
 
@@ -143,5 +209,31 @@ def test_builder_gem_version
     COMMAND
 
     assert_equal expected.strip, command
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_builder_gem_version_self_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
+    pipeline = PipelineFixture.new do
+      use Buildkite::Config::DockerBuild
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        builder Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("1.9.3"))
+      end
+    end
+
+    plugins = pipeline.to_h["steps"][0]["plugins"]
+
+    compose = plugins.find { |plugin|
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
+
+    assert_equal "ruby-1-9-3-local", compose["image-name"]
+    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"]
+    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"]
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end
 end
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index 0e72a909..7937c89c 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -175,11 +175,11 @@ def test_artifacts
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     artifacts = plugins.select { |plugin|
-      plugin.key?("artifacts#v1.0")
+      plugin.key?(plugins_map[:artifacts])
     }
-    assert_equal ".dockerignore", artifacts[0]["artifacts#v1.0"]["download"]
+    assert_equal ".dockerignore", artifacts[0][plugins_map[:artifacts]]["download"]
 
-    download = artifacts[1]["artifacts#v1.0"]
+    download = artifacts[1][plugins_map[:artifacts]]
     assert_equal %w[
       .buildkite/.empty
       .buildkite/docker-compose.yml
@@ -191,7 +191,44 @@ def test_artifacts
     assert_equal ".buildkite.tgz", download["compressed"]
   end
 
-  def test_compose
+  def test_compose_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "hosted"
+
+    pipeline = PipelineFixture.new do
+      build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.2"))
+      use Buildkite::Config::RakeCommand
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        rake "test", task: "test:all"
+      end
+    end
+
+    plugins = pipeline.to_h["steps"][0]["plugins"]
+
+    compose = plugins.find { |plugin|
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
+
+    %w[env run config shell tty].each do |key|
+      assert_includes compose, key
+    end
+
+    assert_includes compose["env"], "PRE_STEPS"
+    assert_includes compose["env"], "RACK"
+
+    assert_equal "default", compose["run"]
+    assert_equal "true", compose["tty"]
+    assert_equal ".buildkite/docker-compose.yml", compose["config"]
+    assert_equal ["runner", "test"], compose["shell"]
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
+  end
+
+  def test_compose_self_hosted
+    @before_env_compute_type = ENV["BUILDKITE_COMPUTE_TYPE"]
+    ENV["BUILDKITE_COMPUTE_TYPE"] = "self-hosted"
+
     pipeline = PipelineFixture.new do
       build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.2"))
       use Buildkite::Config::RakeCommand
@@ -204,10 +241,10 @@ def test_compose
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
 
-    %w[env run config shell].each do |key|
+    %w[env run pull config shell].each do |key|
       assert_includes compose, key
     end
 
@@ -215,8 +252,11 @@ def test_compose
     assert_includes compose["env"], "RACK"
 
     assert_equal "default", compose["run"]
+    assert_equal "default", compose["pull"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]
+  ensure
+    ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end
 
   def test_multiple
@@ -239,8 +279,8 @@ def test_multiple
       plugins = pipeline.to_h["steps"][index]["plugins"]
 
       compose = plugins.find { |plugin|
-        plugin.key?("docker-compose#v1.0")
-      }.fetch("docker-compose#v1.0")
+        plugin.key?(plugins_map[:compose])
+      }.fetch(plugins_map[:compose])
 
       assert_equal "default", compose["run"]
     end
@@ -259,8 +299,8 @@ def test_docker_compose_plugin_service
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
 
     %w[run].each do |key|
       assert_includes compose, key
@@ -309,8 +349,8 @@ def test_env_pre_steps
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
 
     assert_includes compose["env"], "PRE_STEPS"
   end
@@ -388,8 +428,8 @@ def test_rake_env_kwarg
     plugins = pipeline.to_h["steps"][0]["plugins"]
 
     compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+      plugin.key?(plugins_map[:compose])
+    }.fetch(plugins_map[:compose])
 
     assert_includes compose["env"], "RACK"
   end
@@ -431,12 +471,12 @@ def test_bundle_command
 
     artifacts = plugins[0]
 
-    assert_equal "artifacts#v1.0", artifacts.keys.first
-    assert_equal ".dockerignore", artifacts["artifacts#v1.0"]["download"]
+    assert_equal plugins_map[:artifacts], artifacts.keys.first
+    assert_equal ".dockerignore", artifacts[plugins_map[:artifacts]]["download"]
 
     artifacts = plugins[1]
 
-    assert_equal "artifacts#v1.0", artifacts.keys.first
+    assert_equal plugins_map[:artifacts], artifacts.keys.first
     assert_equal %w[
       .buildkite/.empty
       .buildkite/docker-compose.yml
@@ -444,10 +484,10 @@ def test_bundle_command
       .buildkite/Dockerfile.beanstalkd
       .buildkite/mysql-initdb.d
       .buildkite/runner
-    ], artifacts["artifacts#v1.0"]["download"]
-    assert_equal ".buildkite.tgz", artifacts["artifacts#v1.0"]["compressed"]
+    ], artifacts[plugins_map[:artifacts]]["download"]
+    assert_equal ".buildkite.tgz", artifacts[plugins_map[:artifacts]]["compressed"]
 
-    compose = plugins[2].fetch("docker-compose#v1.0")
+    compose = plugins[2].fetch(plugins_map[:compose])
 
     assert_not_includes compose, "env"
     assert_equal "default", compose["run"]
diff --git a/test/test_helper.rb b/test/test_helper.rb
index c59f12bc..eda5c886 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -9,19 +9,31 @@
 require "pathname"
 BUILDKITE_CONFIG_ROOT = Pathname.new(File.expand_path("../..", __dir__))
 
+module TestHelpers
+  def plugins_map
+    {
+      compose: "docker-compose#v5.4.1",
+      artifacts: "artifacts#v1.9.3"
+    }
+  end
+end
+
 class PipelineFixture < Buildkite::Builder::Pipeline
+  include TestHelpers
+
   def initialize(root = BUILDKITE_CONFIG_ROOT, logger: nil, &block)
     @pipeline_definition = Proc.new(&block)
     super(root, logger: logger)
     use(Buildkite::Config::BuildContext)
 
     plugins = extensions.find(Buildkite::Builder::Extensions::Plugins)
-    plugins.manager.add :docker_compose, "docker-compose#v1.0"
-    plugins.manager.add :artifacts, "artifacts#v1.0"
+    plugins.manager.add :docker_compose, plugins_map[:compose]
+    plugins.manager.add :artifacts, plugins_map[:artifacts]
   end
 end
 
 class TestCase < ActiveSupport::TestCase
+  include TestHelpers
   make_my_diffs_pretty!
 
   def setup

From 9bcf6ad86316dc8294b7e7d28644299e2ffb2a9a Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 13:00:06 +0900
Subject: [PATCH 08/15] Use docker-compose plugin to v5.6.0

https://buildkite.com/rails/rails-ci/builds/284
---
 lib/buildkite/config/rake_command.rb       | 5 ++---
 pipelines/rails-ci/pipeline.rb             | 2 +-
 test/buildkite_config/test_rake_command.rb | 3 ++-
 test/test_helper.rb                        | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index 2be65add..ecaf090e 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -60,11 +60,10 @@ def install_plugins(service = "default", env = nil, dir = ".", build_context:)
           "run" => service,
           "config" => ".buildkite/docker-compose.yml",
           "shell" => ["runner", *dir],
+          "tty" => "true",
         }
 
-        if build_context.hosted?
-          compose_opts["tty"] = "true"
-        else
+        if build_context.self_hosted?
           compose_opts["pull"] = service
           compose_opts["pull-retries"] = 3
         end
diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb
index 3e1c03bf..36236e37 100644
--- a/pipelines/rails-ci/pipeline.rb
+++ b/pipelines/rails-ci/pipeline.rb
@@ -7,7 +7,7 @@
   use Buildkite::Config::RakeCommand
   use Buildkite::Config::RubyGroup
 
-  plugin :docker_compose, "docker-compose#v5.4.1"
+  plugin :docker_compose, "docker-compose#v5.6.0"
   plugin :artifacts, "artifacts#v1.9.3"
   plugin :secrets, "cluster-secrets#v1.0.0"
 
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index 7937c89c..cbee1b24 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -244,7 +244,7 @@ def test_compose_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    %w[env run pull config shell].each do |key|
+    %w[env run pull config shell tty].each do |key|
       assert_includes compose, key
     end
 
@@ -253,6 +253,7 @@ def test_compose_self_hosted
 
     assert_equal "default", compose["run"]
     assert_equal "default", compose["pull"]
+    assert_equal "true", compose["tty"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]
   ensure
diff --git a/test/test_helper.rb b/test/test_helper.rb
index eda5c886..07738ae9 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -12,7 +12,7 @@
 module TestHelpers
   def plugins_map
     {
-      compose: "docker-compose#v5.4.1",
+      compose: "docker-compose#v5.6.0",
       artifacts: "artifacts#v1.9.3"
     }
   end

From bf42f52633b33aab4243bd90df7cdba432868249 Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 13:43:35 +0900
Subject: [PATCH 09/15] Use docker-compose v1 in self-hosted

---
 lib/buildkite/config/rake_command.rb       | 1 +
 test/buildkite_config/test_rake_command.rb | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index ecaf090e..0c820410 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -64,6 +64,7 @@ def install_plugins(service = "default", env = nil, dir = ".", build_context:)
         }
 
         if build_context.self_hosted?
+          compose_opts["cli-version"] = "1"
           compose_opts["pull"] = service
           compose_opts["pull-retries"] = 3
         end
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index cbee1b24..7edb4bf8 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -244,7 +244,7 @@ def test_compose_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    %w[env run pull config shell tty].each do |key|
+    %w[env run cli-version pull pull-retries config shell tty].each do |key|
       assert_includes compose, key
     end
 
@@ -252,7 +252,9 @@ def test_compose_self_hosted
     assert_includes compose["env"], "RACK"
 
     assert_equal "default", compose["run"]
+    assert_equal "1", compose["cli-version"]
     assert_equal "default", compose["pull"]
+    assert_equal 3, compose["pull-retries"]
     assert_equal "true", compose["tty"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]

From a4cb57e695f0c77a8066a227cd4615b6c9c6d550 Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 14:05:20 +0900
Subject: [PATCH 10/15] Turn off pull for self-hosted compose runs

---
 lib/buildkite/config/rake_command.rb       | 2 --
 test/buildkite_config/test_rake_command.rb | 4 +---
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index 0c820410..a315b634 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -65,8 +65,6 @@ def install_plugins(service = "default", env = nil, dir = ".", build_context:)
 
         if build_context.self_hosted?
           compose_opts["cli-version"] = "1"
-          compose_opts["pull"] = service
-          compose_opts["pull-retries"] = 3
         end
 
         plugin :docker_compose, compose_opts.compact
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index 7edb4bf8..0311c1e4 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -244,7 +244,7 @@ def test_compose_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    %w[env run cli-version pull pull-retries config shell tty].each do |key|
+    %w[env run cli-version config shell tty].each do |key|
       assert_includes compose, key
     end
 
@@ -253,8 +253,6 @@ def test_compose_self_hosted
 
     assert_equal "default", compose["run"]
     assert_equal "1", compose["cli-version"]
-    assert_equal "default", compose["pull"]
-    assert_equal 3, compose["pull-retries"]
     assert_equal "true", compose["tty"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]

From e6651bc9ead220fceb0cbb0fc3daa63c47097115 Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 14:46:08 +0900
Subject: [PATCH 11/15] Remove image-repository

---
 lib/buildkite/config/docker_build.rb       | 1 -
 test/buildkite_config/test_docker_build.rb | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb
index a70486d6..bae9ec98 100644
--- a/lib/buildkite/config/docker_build.rb
+++ b/lib/buildkite/config/docker_build.rb
@@ -85,7 +85,6 @@ def builder(ruby)
               "image-name" => build_context.ruby.image_name_for(build_context.build_id),
               "cache-from" => cache_from(build_context),
               push: build_push(build_context),
-              "image-repository" => build_context.image_base,
             }
           end
 
diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb
index 3b5372a0..296cbee2 100644
--- a/test/buildkite_config/test_docker_build.rb
+++ b/test/buildkite_config/test_docker_build.rb
@@ -94,7 +94,7 @@ def test_builder_compose_plugin_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    %w[image-name cache-from push build config env image-repository].each do |key|
+    %w[image-name cache-from push build config env].each do |key|
       assert_includes compose, key
     end
     assert_equal "3-2-local", compose["image-name"]
@@ -105,7 +105,6 @@ def test_builder_compose_plugin_self_hosted
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_includes compose["env"], "PRE_STEPS"
     assert_includes compose["env"], "RACK"
-    assert_equal "buildkite-config-base", compose["image-repository"]
   ensure
     ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end

From e56eff4e4358087ec082a24cde6c2087dd519a3e Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 15:00:05 +0900
Subject: [PATCH 12/15] Update minitest to get proper locations on Ruby 3.4

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 01bc3fef..219179e0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -40,7 +40,7 @@ GEM
       reline (>= 0.3.8)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    minitest (5.19.0)
+    minitest (5.25.4)
     mutex_m (0.1.2)
     parallel (1.23.0)
     parser (3.2.2.4)

From e3be689145a2ac7ed34514e63c841dd545453abb Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 15:01:36 +0900
Subject: [PATCH 13/15] Remove image-name from docker build

---
 lib/buildkite/config/docker_build.rb       | 1 -
 test/buildkite_config/test_docker_build.rb | 4 +---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb
index bae9ec98..f454082f 100644
--- a/lib/buildkite/config/docker_build.rb
+++ b/lib/buildkite/config/docker_build.rb
@@ -82,7 +82,6 @@ def builder(ruby)
               build: "base",
               config: ".buildkite/docker-compose.yml",
               env: %w[PRE_STEPS RACK],
-              "image-name" => build_context.ruby.image_name_for(build_context.build_id),
               "cache-from" => cache_from(build_context),
               push: build_push(build_context),
             }
diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb
index 296cbee2..9b4d6e73 100644
--- a/test/buildkite_config/test_docker_build.rb
+++ b/test/buildkite_config/test_docker_build.rb
@@ -94,10 +94,9 @@ def test_builder_compose_plugin_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    %w[image-name cache-from push build config env].each do |key|
+    %w[cache-from push build config env].each do |key|
       assert_includes compose, key
     end
-    assert_equal "3-2-local", compose["image-name"]
     assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"]
     assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"]
 
@@ -229,7 +228,6 @@ def test_builder_gem_version_self_hosted
       plugin.key?(plugins_map[:compose])
     }.fetch(plugins_map[:compose])
 
-    assert_equal "ruby-1-9-3-local", compose["image-name"]
     assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"]
     assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"]
   ensure

From e8ece5f5fe090481ae3443385855c6f29529af38 Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 15:11:22 +0900
Subject: [PATCH 14/15] Pass BUILDKITE_BUILD_ID into diff/annotate

This should allow to see, for example, the expected image tag being
built to carry over.

```diff
-    - docker-compose#v4.16.0:
+    - docker-compose#v5.0.0:
         build: base
         config: ".buildkite/docker-compose.yml"
         env:
         - PRE_STEPS
         - RACK
-        image-name: ruby-3-4-build_id
         cache-from:
         - base:973266071021.dkr.ecr.us-east-1.amazonaws.com/builds:ruby-3-4-br-main
         push:
         - base:973266071021.dkr.ecr.us-east-1.amazonaws.com/builds:ruby-3-4-br-
-        image-repository: 973266071021.dkr.ecr.us-east-1.amazonaws.com/builds
```

Notice how the tag is only `ruby-3-4-br-` because the build id was
missing from the environment when generating the pipeline.
---
 pipelines/buildkite-config/initial.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pipelines/buildkite-config/initial.yml b/pipelines/buildkite-config/initial.yml
index 0158f130..95dec142 100644
--- a/pipelines/buildkite-config/initial.yml
+++ b/pipelines/buildkite-config/initial.yml
@@ -57,6 +57,7 @@ steps:
           -v "$$PWD":/app:ro -w /app
           -v "$$PWD/tmp":/app/tmp:rw
           -e BUNDLE_FROZEN
+          -e BUILDKITE_BUILD_ID
           ruby:latest
           ./bin/pipeline-annotate
     - command: |
@@ -80,6 +81,7 @@ steps:
           -v "$$PWD/tmp":/app/tmp:rw
           -e RAILS_CI_NIGHTLY
           -e BUNDLE_FROZEN
+          -e BUILDKITE_BUILD_ID
           ruby:latest
           ./bin/pipeline-annotate
 

From 89ddf2003382a8bd9fc3862989a2eea0c2125a3d Mon Sep 17 00:00:00 2001
From: zzak <zzak@hey.com>
Date: Mon, 10 Mar 2025 15:19:50 +0900
Subject: [PATCH 15/15] Include build_id in push

---
 lib/buildkite/config/docker_build.rb       | 1 +
 test/buildkite_config/test_docker_build.rb | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb
index f454082f..ecbde66f 100644
--- a/lib/buildkite/config/docker_build.rb
+++ b/lib/buildkite/config/docker_build.rb
@@ -37,6 +37,7 @@ def build_push(build_context)
             build_context.local_branch =~ /:/ ?
               build_context.image_name_for("pr-#{build_context.pull_request}") :
               build_context.image_name_for("br-#{build_context.local_branch}"),
+            build_context.image_name_for(build_context.build_id)
           ]
         end
       end
diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb
index 9b4d6e73..aa998689 100644
--- a/test/buildkite_config/test_docker_build.rb
+++ b/test/buildkite_config/test_docker_build.rb
@@ -98,7 +98,7 @@ def test_builder_compose_plugin_self_hosted
       assert_includes compose, key
     end
     assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"]
+    assert_equal ["base:buildkite-config-base:3-2-br-", "base:buildkite-config-base:3-2-local"], compose["push"]
 
     assert_equal "base", compose["build"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
@@ -229,7 +229,7 @@ def test_builder_gem_version_self_hosted
     }.fetch(plugins_map[:compose])
 
     assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"]
+    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-", "base:buildkite-config-base:ruby-1-9-3-local"], compose["push"]
   ensure
     ENV["BUILDKITE_COMPUTE_TYPE"] = @before_env_compute_type
   end