wal-e ssh_strategy fork is stable; build into base image

Author: drnic

.gitmodules

@@ -1,4 +1,4 @@
-[submodule "wal-e"]
-	path = wal-e
+[submodule "images/postgresql95-base/wal-e"]
+	path = images/postgresql95-base/wal-e
 	url = https://github.com/drnic/wal-e.git
 	branch = ssh_strategy

Dockerfile

@@ -16,9 +16,6 @@ RUN echo "[[ -f /etc/patroni.d/.envrc ]] && source /etc/patroni.d/.envrc" >> /ro
 COPY images/supervisord.conf /etc/supervisor/supervisord.conf
 COPY images/services/*.conf /etc/supervisor/conf.d/
 
-COPY wal-e /tmp/wal-e
-RUN cd /tmp/wal-e && pip3 install . --upgrade
-
 COPY images/scripts /scripts
 CMD ["/scripts/entry.sh"]
 

ci/pipeline.yml

@@ -32,12 +32,9 @@ jobs:
       - {get: agent-ci}
       - {get: agent, passed: [rc]}
       - {get: version, passed: [rc], trigger: true}
-      - {get: wal-e, resource: wale-fork, trigger: true}
-    - task: custom-wal-e
-      file: agent-ci/ci/tasks/custom-wal-e.yml
     - put: pg95-image
       params:
-        build: dockerfile
+        build: agent
         # cache: true -- causing https://ci.starkandwayne.com/teams/main/pipelines/dingo-postgresql-agent/jobs/build-image/builds/16
         tag: agent-ci/ci/tags/pre-test
 
@@ -140,11 +137,16 @@ jobs:
   public: true
   serial: true
   plan:
-  - {get: agent-base, trigger: true}
-  - {get: agent}
+  - aggregate:
+    - {get: agent, resource: agent-base, trigger: true}
+    - {get: agent-ci}
+    - {get: wal-e, resource: wale-fork, trigger: true}
+  - task: custom-wal-e
+    file: agent-ci/ci/tasks/custom-wal-e.yml
+    params: {dockerfile_dir: agent/images/postgresql95-base}
   - put: pg95-base-image
     params:
-      build: agent/images/postgresql95-base
+      build: dockerfile
       cache: true
 
 - name: build-task-image

ci/tasks/custom-wal-e.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-cp -r agent/* dockerfile/
+cp -r ${dockerfile_dir}/* dockerfile/
 
 # remove submodule from development
 rm -rf dockerfile/wal-e

ci/tasks/custom-wal-e.yml

@@ -11,5 +11,8 @@ inputs:
 outputs:
 - name: dockerfile
 
+params:
+  dockerfile_dir:
+
 run:
   path: agent-ci/ci/tasks/custom-wal-e.sh

images/postgresql95-base/Dockerfile

@@ -72,6 +72,10 @@ RUN set -x \
       && rm -rf /var/cache/apk/* \
       && pip3 install --no-cache-dir wal-e[aws,azure,google,swift]==${WALE_VERSION} awscli envdir pgxnclient --upgrade
 
+# Now install patched wal-e with ssh_strategy
+COPY wal-e /tmp/wal-e
+RUN cd /tmp/wal-e && pip3 install . --upgrade
+
 # python2, dumb-init, supervisor
 ENV SUPERVISOR_VERSION=3.3.1
 RUN set -x \

images/postgresql95-base/README.md

@@ -3,7 +3,7 @@
 To see the versioned aspects of the base Docker image, run:
 
 ```
-grep "^ENV.*VERSION" images/pg96-base/Dockerfile
+grep "^ENV.*VERSION" images/postgresql95-base/Dockerfile
 ```
 
 The output will look similar to:
@@ -27,7 +27,7 @@ docker build -t dingotiles/dingo-postgresql95-agent-base:latest images/postgresq
 Sanity check of installed tools:
 
 ```
-$ image=dingotiles/dingo-postgresql96-agent-base:latest
+$ image=dingotiles/dingo-postgresql95-agent-base:latest
 
 $ docker run -ti $image postgres --version
 postgres (PostgreSQL) 9.5.4
@@ -38,7 +38,7 @@ jq-1.5
 $ docker run -ti $image etcdctl --version
 etcdctl version 2.3.4
 $ docker run -ti $image wal-e version
-1.0.2
+1.1.dev0
 $ docker run -ti $image python --version
 Python 2.7.12
 $ docker run -ti $image python3 --version