Integrate revng-distributable tests with orchestra

Author: Filippo Cremonese

.orchestra/ci/install-dependencies.sh

@@ -47,6 +47,12 @@ apt-get -qq install --no-install-recommends --yes \
   wget \
   zlib1g-dev
 
+# Dependencies for testing cold-revng
+apt-get install -y \
+  fluxbox \
+  xvfb \
+  wmctrl \
+
 # Dependencies for Qt
 apt-get -qq install --no-install-recommends --yes \
   gperf \

.orchestra/config/components.yml

@@ -22,6 +22,7 @@ environment:
   - HARD_FLAGS_LINK: #@ str(datavalue("hard_flags_link"))
   - HARD_FLAGS_LINK_LATE: #@ str(datavalue("hard_flags_link_late"))
   - HARD_FLAGS_LINK_GOLD: #@ str(datavalue("hard_flags_link_gold"))
+  - COLD_REVNG_TEST_METHOD: #@ str(datavalue("cold_revng_test_method"))
   - "-LD_LIBRARY_PATH": ""
   - "-COMPILER_PATH": ""
   - "-CPATH": ""

.orchestra/config/components/revng_distributable.yml

@@ -1,6 +1,12 @@
 #@ load("@ytt:overlay", "overlay")
 #@ load("/lib/create_component.lib.yml", "single_build_component")
 
+#! Possible values for the cold_revng_test_method user option:
+#! - host-xvfb: run the tests on the host machine, using xvfb and fluxbox
+#! - host-x11: run the tests on the host machine, using the host running X11 and window manager
+#! - podman-xvbf: run the tests on all the distros using podman and xvfb
+#! - podman-forward-x11: run the tests on all the distros, forwarding the host X11 server
+
 #@yaml/text-templated-strings
 ---
 #@ def revng_distributable():
@@ -132,6 +138,55 @@ install: |
   cd "${TMP_ROOT}${ORCHESTRA_ROOT}"
   find . -not -type d -not -path './revng/*' -delete
   find . -type d -empty -delete
+
+  if test "$RUN_TESTS" -eq 1; then
+    # CI can't run Docker-in-Docker (or Podman-in-Docker)
+    # We can only test by spawning a local Xvfb server
+    if [[ ! -z "${GITLAB_CI:-}" ]]; then
+      export USE_XVFB=1
+      cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng"
+      "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh"
+
+    elif [[ "$COLD_REVNG_TEST_METHOD" == host-* ]]; then
+      if [[ "$COLD_REVNG_TEST_METHOD" == "host-xvfb" ]]; then
+        export USE_XVFB=1
+      fi
+      cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng"
+
+      export COLD_REVNG_KILL_METHOD=wmctrl
+      "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh"
+
+    else
+      if [[ "$COLD_REVNG_TEST_METHOD" == "podman-forward-x11" ]]; then
+        # Remember: the host must allow access to the X11 socket! (xhost +)
+        PODMAN_OPTIONS="-v /tmp/.X11-unix:/tmp/.X11-unix"
+      else
+        PODMAN_OPTIONS="-e USE_XVFB=1"
+      fi
+
+      for DOCKERFILE in $ORCHESTRA_DOTDIR/support/revng-distributable/dockers/*.docker; do
+        DISTRO="$(basename "${DOCKERFILE%%.docker}")"
+        IMAGE_NAME="revng-test-image:$DISTRO"
+
+        echo "Testing revng-distributable in $DISTRO"
+        podman build --pull -t "$IMAGE_NAME" -f "$DOCKERFILE"
+        if ! podman run -t --rm $PODMAN_OPTIONS \
+          --security-opt=seccomp=unconfined \
+          -e "COLD_REVNG_KILL_METHOD=kill" \
+          -w "/orchestra/.orchestra/$(realpath --relative-to "$ORCHESTRA_DOTDIR" "$TMP_ROOT")/$ORCHESTRA_ROOT/revng" \
+          -v "$(readlink -f "$ORCHESTRA_DOTDIR/..")":/orchestra \
+          "$IMAGE_NAME" \
+          bash -c /orchestra/.orchestra/support/revng-distributable/test.sh;
+        then
+          echo "[!] $IMAGE_NAME test failed"
+          exit 1
+        else
+          echo "[+] $IMAGE_NAME test passed"
+        fi
+      done
+      echo "[+] revng-distributable tests passed"
+    fi
+  fi
 build_dependencies:
   - ui/cold-revng
 skip_post_install: true

.orchestra/config/global_options.yml

@@ -36,3 +36,6 @@ branches:
   - develop
   - master
 nonredistributable_base_url: "https://dummydomain.com/nonredistributable/"
+
+#! see cold-revng component for the list of admissible test methods
+cold_revng_test_method: podman-xvfb

.orchestra/support/revng-distributable/dockers/README.md

@@ -0,0 +1,6 @@
+# Status of revng-distributable tests
+
+## Broken distros
+
+- gentoo: need to revise the scripts which install dependencies
+- opensuse 15.{0,1}: zypper segfaults

.orchestra/support/revng-distributable/dockers/archlinux.docker

@@ -1,7 +1,8 @@
 FROM archlinux
 
-COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+COPY install-revng-dependencies install-test-dependencies /
+RUN /install-revng-dependencies && \
+    /install-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/broken/gentoo.docker

@@ -0,0 +1,15 @@
+FROM gentoo/stage3-amd64 AS base-gentoo
+
+COPY install-revng-dependencies install-test-dependencies /
+RUN echo 'FEATURES="$FEATURES -sandbox -usersandbox -ipc-sandbox -network-sandbox -pid-sandbox"' >> /etc/portage/make.conf
+RUN emerge-webrsync
+
+FROM base-gentoo AS gentoo-with-revng-dependencies
+RUN /install-revng-dependencies
+
+FROM gentoo-with-revng-dependencies
+RUN echo "sys-auth/elogind" >> "/etc/portage/package.mask"
+RUN /install-test-dependencies
+
+ENV DISPLAY=:0
+ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/broken/opensuse-15.0.docker

@@ -0,0 +1,8 @@
+FROM opensuse/leap:15.0
+
+COPY install-revng-dependencies install-test-dependencies /
+RUN /install-revng-dependencies && \
+    /install-test-dependencies
+
+ENV DISPLAY=:0
+ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/broken/opensuse-15.1.docker

@@ -0,0 +1,8 @@
+FROM opensuse/leap:15.1
+
+COPY install-revng-dependencies install-test-dependencies /
+RUN /install-revng-dependencies && \
+    /install-test-dependencies
+
+ENV DISPLAY=:0
+ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/build-all

@@ -3,8 +3,10 @@
 set -x
 set -e
 
-FILES=$(find -name "*.docker")
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+FILES=$(find "$DIR" -name "*.docker")
 
 for FILE in $FILES; do
-    ./build $(basename "${FILE%.*}")
+    FILE_PATH=$(realpath "$FILE")
+    "$DIR/build" "$(basename "${FILE%.*}")"
 done