Build & CI fixes

.circleci/config.yml

@@ -3,41 +3,39 @@ version: 2.1
 jobs:
   build:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
       - add_ssh_keys:
           fingerprints:
             - "f9:fc:d7:c4:a9:17:3d:09:ee:03:d7:f9:6e:0d:f5:2b"
       - run:
-          name: Write git tag name to text file "version"
-          command: git describe --tags --always > version
-      - run:
-          name: Build Docker container
-          command: docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} .
-          no_output_timeout: 6h
-      - run:
-          name: Save built Docker container to cache
-          command: mkdir -p ~/docker; docker save "bids/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar
-          no_output_timeout: 6h
+          name: Build Docker image
+          command: |
+            wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/build_docker.sh
+            bash build_docker.sh
+            no_output_timeout: 6h
       - save_cache:
           key: image_cache-{{ .Revision }}-{{ epoch }}
           paths:
             - "~/docker"
 
-
   test_preproc:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
       - restore_cache:
           key: image_cache-{{ .Revision }}
       - run:
           name: Download test data
-          command: mkdir -p ${HOME}/downloads && wget -c -O ${HOME}/downloads/preproc.tar.gz "https://osf.io/d9gx7/download" && mkdir -p ${HOME}/data && tar xf ${HOME}/downloads/preproc.tar.gz -C ${HOME}/data
+          command:
+            mkdir -p ${HOME}/downloads && wget -c -O
+            ${HOME}/downloads/preproc.tar.gz "https://osf.io/d9gx7/download" &&
+            mkdir -p ${HOME}/data && tar xf ${HOME}/downloads/preproc.tar.gz -C
+            ${HOME}/data
       - run:
           name: Make scratch directory
           command: mkdir ${HOME}/scratch
@@ -46,24 +44,33 @@ jobs:
           command: docker load -i ~/docker/image.tar
       - run:
           name: Print version
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp bids/${CIRCLE_PROJECT_REPONAME,,} --version
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp
+            bids/${CIRCLE_PROJECT_REPONAME,,} --version
       - run:
           name: Run preproc-level analysis for all subjects
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs preproc --scratch /scratch
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v
+            ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v
+            ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,}
+            /bids_dataset /outputs preproc --scratch /scratch
           no_output_timeout: 6h
 
-
   test_participant:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
       - restore_cache:
           key: image_cache-{{ .Revision }}
       - run:
           name: Download test data
-          command: mkdir -p ${HOME}/downloads && wget -c -O ${HOME}/downloads/participant.tar.gz "https://osf.io/skc5x/download" && mkdir -p ${HOME}/data && tar xf ${HOME}/downloads/participant.tar.gz -C ${HOME}/data
+          command:
+            mkdir -p ${HOME}/downloads && wget -c -O
+            ${HOME}/downloads/participant.tar.gz "https://osf.io/skc5x/download"
+            && mkdir -p ${HOME}/data && tar xf
+            ${HOME}/downloads/participant.tar.gz -C ${HOME}/data
       - run:
           name: Make scratch directory
           command: mkdir ${HOME}/scratch
@@ -72,24 +79,34 @@ jobs:
           command: docker load -i ~/docker/image.tar
       - run:
           name: Print version
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp bids/${CIRCLE_PROJECT_REPONAME,,} --version
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp
+            bids/${CIRCLE_PROJECT_REPONAME,,} --version
       - run:
           name: Run participant-level analysis for solitary subject
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --scratch /scratch --parcellation aal --streamlines 100000 --output_verbosity 3
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v
+            ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v
+            ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,}
+            /bids_dataset /outputs participant --scratch /scratch --parcellation
+            aal --streamlines 100000 --output_verbosity 3
           no_output_timeout: 6h
 
-
   test_group:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
       - restore_cache:
           key: image_cache-{{ .Revision }}
       - run:
           name: Download test data
-          command: mkdir -p ${HOME}/downloads && wget -c -O ${HOME}/downloads/group.tar.gz "https://osf.io/6s8bx/download" && mkdir -p ${HOME}/data && tar xf ${HOME}/downloads/group.tar.gz -C ${HOME}/data
+          command:
+            mkdir -p ${HOME}/downloads && wget -c -O
+            ${HOME}/downloads/group.tar.gz "https://osf.io/6s8bx/download" &&
+            mkdir -p ${HOME}/data && tar xf ${HOME}/downloads/group.tar.gz -C
+            ${HOME}/data
       - run:
           name: Make scratch directory
           command: mkdir ${HOME}/scratch
@@ -98,16 +115,21 @@ jobs:
           command: docker load -i ~/docker/image.tar
       - run:
           name: Print version
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp bids/${CIRCLE_PROJECT_REPONAME,,} --version
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp
+            bids/${CIRCLE_PROJECT_REPONAME,,} --version
       - run:
           name: Run group-level analysis
-          command: docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group --scratch /scratch
+          command:
+            docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v
+            ${HOME}/data/BIDS:/bids_dataset -v ${HOME}/data:/outputs -v
+            ${HOME}/scratch:/scratch bids/${CIRCLE_PROJECT_REPONAME,,}
+            /bids_dataset /outputs group --scratch /scratch
           no_output_timeout: 6h
 
-
   deploy:
-    docker:
-      - image: circleci/classic:latest
+    machine:
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
@@ -117,16 +139,12 @@ jobs:
           name: Load Docker image
           command: docker load -i ~/docker/image.tar
       - run:
-          name: Push "latest" tag to DockerHub
-          command: if [[ -n "$DOCKER_PASS" ]]; then docker login -u $DOCKER_USER -p $DOCKER_PASS && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:latest; fi
-          no_output_timeout: 6h
-      - run:
-          name: Create named tag and push to DockerHub
-          command: if [[ -n "$DOCKER_PASS" ]]; then docker login -u $DOCKER_USER -p $DOCKER_PASS && docker tag bids/${CIRCLE_PROJECT_REPONAME,,} bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG; fi
-          no_output_timeout: 6h
+          name: push to dockerhub
+          command: |
+            wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/push_docker.sh
+            bash push_docker.sh
 
 workflows:
-  version: 2.1
   build_test_deploy:
     jobs:
       - build:
@@ -160,13 +178,13 @@ workflows:
             branches:
               only: /.*/
       - deploy:
+          context:
+          - dockerhub
           requires:
             - build
             - test_preproc
             - test_participant
             - test_group
           filters:
-            branches:
-              ignore: /.*/
             tags:
               only: /.*/

.hadolint.yaml

@@ -0,0 +1,8 @@
+# config file for linting of Dockerfiles by hadolint
+---
+ignored:
+- DL3003
+- DL3008
+- DL4001
+- DL4006
+- SC2015

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+---
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+repos:
+
+- repo: https://github.com/hadolint/hadolint
+  rev: v2.12.0
+  hooks:
+  - id: hadolint-docker
+    name: Lint Dockerfiles
+    description: Runs hadolint Docker image to lint Dockerfiles
+    language: docker_image
+    types: [dockerfile]
+    entry: ghcr.io/hadolint/hadolint hadolint
+
+ci:
+  skip: [hadolint-docker]

Dockerfile

@@ -1,45 +1,59 @@
 FROM ubuntu:18.04
-MAINTAINER Robert E. Smith <robert.smith@florey.edu.au>
+
+# MAINTAINER Robert E. Smith <robert.smith@florey.edu.au>
+
+ARG DEBIAN_FRONTEND="noninteractive"
 
 # Core system capabilities required
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    bc \
-    build-essential \
-    curl \
-    dc \
-    git \
-    libegl1-mesa-dev \
-    libopenblas-dev \
-    nano \
-    perl-modules-5.26 \
-    python2.7 \
-    python3 \
-    tar \
-    tcsh \
-    tzdata \
-    unzip \
-    wget
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        bc \
+        build-essential \
+        curl \
+        dc \
+        git \
+        libegl1-mesa-dev \
+        libopenblas-dev \
+        nano \
+        perl-modules-5.26 \
+        python2.7 \
+        python3 \
+        tar \
+        tcsh \
+        tzdata \
+        unzip \
+        ca-certificates \
+        apt-utils \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # PPA for newer version of nodejs, which is required for bids-validator
-RUN curl -sL https://deb.nodesource.com/setup_12.x -o nodesource_setup.sh && \
-    bash nodesource_setup.sh && \
-    rm -f nodesource_setup.sh && \
-    apt-get install -y nodejs
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update -qq && \
+    apt-get install -y -q --no-install-recommends \
+                  nodejs && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN node --version && npm --version && npm install -g bids-validator@1.5.3
+
 
 # NeuroDebian setup
 COPY neurodebian.gpg /neurodebian.gpg
 RUN wget -qO- http://neuro.debian.net/lists/bionic.au.full | \
     tee /etc/apt/sources.list.d/neurodebian.sources.list && \
-    apt-key add /neurodebian.gpg && \
-    apt-get update
+    apt-key add /neurodebian.gpg
 
 # Additional dependencies for MRtrix3 compilation
-RUN apt-get update && apt-get install -y \
-    libeigen3-dev \
-    libfftw3-dev \
-    libpng-dev \
-    libtiff5-dev \
-    zlib1g-dev
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        libeigen3-dev \
+        libfftw3-dev \
+        libpng-dev \
+        libtiff5-dev \
+        zlib1g-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Neuroimaging software / data dependencies
 RUN wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.1.1/freesurfer-linux-centos8_x86_64-7.1.1.tar.gz | \
@@ -57,20 +71,24 @@ RUN wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.1.1/frees
     --exclude='freesurfer/lib/cuda' \
     --exclude='freesurfer/lib/qt'
 RUN echo "cHJpbnRmICJyb2JlcnQuc21pdGhAZmxvcmV5LmVkdS5hdVxuMjg1NjdcbiAqQ3FLLjFwTXY4ZE5rXG4gRlNvbGRZRXRDUFZqNlxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
-RUN apt-get install -y ants=2.2.0-1ubuntu1
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+    ants=2.2.0-1ubuntu1 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 # FSL installer appears to now be ready for use with version 6
 # eddy is also now included in FSL6
 RUN wget -q http://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \
     chmod 775 fslinstaller.py && \
-    python2 /fslinstaller.py -d /opt/fsl -V 6.0.4 -q && \
+    python2 /fslinstaller.py -d /opt/fsl -V 6.0.4 && \
     rm -f /fslinstaller.py
 RUN which immv || ( echo "FSLPython not properly configured; re-running" && rm -rf /opt/fsl/fslpython && /opt/fsl/etc/fslconf/fslpython_install.sh -f /opt/fsl || ( cat /tmp/fslpython*/fslpython_miniconda_installer.log && exit 1 ) )
-RUN wget -qO- "https://www.nitrc.org/frs/download.php/5994/ROBEXv12.linux64.tar.gz//?i_agree=1&download_now=1" | \
-    tar zx -C /opt
-RUN npm install -gq bids-validator@1.5.3
+RUN wget -qO- "https://www.nitrc.org/frs/download.php/5994/ROBEXv12.linux64.tar.gz//?i_agree=1&download_now=1" | tar zx -C /opt
+
 
 # apt cleanup to recover as much space as possible
-RUN apt-get remove -y libegl1-mesa-dev && \
+RUN apt-get remove -qq -y \
+        libegl1-mesa-dev && \
     apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*