Integrate revng-distributable tests with orchestra

.orchestra/config/components.yml

@@ -22,6 +22,7 @@ environment:
   - HARD_FLAGS_LINK: #@ str(datavalue("hard_flags_link"))
   - HARD_FLAGS_LINK_LATE: #@ str(datavalue("hard_flags_link_late"))
   - HARD_FLAGS_LINK_GOLD: #@ str(datavalue("hard_flags_link_gold"))
+  - COLD_REVNG_TEST_METHOD: #@ str(datavalue("cold_revng_test_method"))
   - "-LD_LIBRARY_PATH": ""
   - "-COMPILER_PATH": ""
   - "-CPATH": ""

.orchestra/config/components/revng_distributable.yml

@@ -1,6 +1,12 @@
 #@ load("@ytt:overlay", "overlay")
 #@ load("/lib/create_component.lib.yml", "single_build_component")
 
+#! Possible values for the cold_revng_test_method user option:
+#! - host-xvfb: run the tests on the host machine, using xvfb and fluxbox
+#! - host-x11: run the tests on the host machine, using the host running X11 and window manager
+#! - podman-xvbf: run the tests on all the distros using podman and xvfb
+#! - podman-forward-x11: run the tests on all the distros, forwarding the host X11 server
+
 #@yaml/text-templated-strings
 ---
 #@ def revng_distributable():
@@ -13,20 +19,36 @@ install: |
   rm -rf "${TMP_ROOT}${ORCHESTRA_ROOT}/revng/root"
   mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/revng/root"
   cd "$ORCHESTRA_ROOT"
-  index_files=""
-  echo share/orchestra/glibc.idx share/orchestra/cmake.idx share/orchestra/boost.idx share/orchestra/clang-release.idx share/orchestra/toolchain_*.idx | sed 's/ /\n/g' | grep -v host | while read index_file; do
-    if [ -e "$index_file" ]; then
-      cat "$index_file" | \
-        grep -v libgcc_s.so | \
-        grep -v 'libc++.so' | \
-        grep -v 'libc++abi.so' | \
-        grep -v 'llvm-link' | \
-        grep -v 'llc'
-    fi
-  done | sed 's!^!/!' | rsync \
+
+  # TODO: This should be derived from orchestra by whitelisting the transitive dependencies of revng-distributable
+  COMPONENTS_TO_EXCLUDE=(
+    glibc
+    boost
+    clang-release
+    cmake
+    llvm-documentation
+  )
+  # Select all installed toolchain components except toolchain/host/*
+  COMPONENTS_TO_EXCLUDE+=(
+    $(orc components --installed --json \
+      | jq -r '
+        .[].name
+        | select(. | test("^toolchain/.*"))
+        | select(. | test("^toolchain/host.*") | not)'
+    )
+  )
+
+  for COMPONENT in "${COMPONENTS_TO_EXCLUDE[@]}"; do
+    orc inspect component installed-files "$COMPONENT" \
+      | grep -v 'libgcc_s.so' \
+      | grep -v 'libc++.so' \
+      | grep -v 'libc++abi.so' \
+      | grep -v 'llvm-link' \
+      | grep -v 'llc'
+  done \
+  | sed 's!^!/!' \
+  | rsync \
     --archive \
-    --verbose \
-    --progress \
     --exclude "/revng/" \
     --exclude "/include/" \
     --exclude "/share/man/" \
@@ -114,6 +136,56 @@ install: |
   cd "${TMP_ROOT}${ORCHESTRA_ROOT}"
   find . -not -type d -not -path './revng/*' -delete
   find . -type d -empty -delete
+
+  if test "$RUN_TESTS" -eq 1; then
+    # CI can't run Docker-in-Docker (or Podman-in-Docker)
+    # We can only test by spawning a local Xvfb server
+    if [[ ! -z "${GITLAB_CI:-}" ]]; then
+      export USE_XVFB=1
+      cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng"
+      "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh"
+
+    elif [[ "$COLD_REVNG_TEST_METHOD" == host-* ]]; then
+      if [[ "$COLD_REVNG_TEST_METHOD" == "host-xvfb" ]]; then
+        export USE_XVFB=1
+      fi
+      cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng"
+
+      export COLD_REVNG_KILL_METHOD=wmctrl
+      "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh"
+
+    else
+      if [[ "$COLD_REVNG_TEST_METHOD" == "podman-forward-x11" ]]; then
+        # Remember: the host must allow access to the X11 socket! (xhost +)
+        PODMAN_OPTIONS="-v /tmp/.X11-unix:/tmp/.X11-unix"
+      else
+        PODMAN_OPTIONS="-e USE_XVFB=1"
+      fi
+
+      for DOCKERFILE in $ORCHESTRA_DOTDIR/support/revng-distributable/dockers/*.docker; do
+        DISTRO="$(basename "${DOCKERFILE%%.docker}")"
+        IMAGE_NAME="revng-test-image:$DISTRO"
+
+        echo "Testing revng-distributable in $DISTRO"
+        podman build --pull -t "$IMAGE_NAME" -f "$DOCKERFILE"
+        # TODO: fix the test environment so that we can properly close revng-ui without using kill
+        if ! podman run -t --rm $PODMAN_OPTIONS \
+          --security-opt=seccomp=unconfined \
+          -e "COLD_REVNG_KILL_METHOD=kill" \
+          -w "/orchestra/.orchestra/$(realpath --relative-to "$ORCHESTRA_DOTDIR" "$TMP_ROOT")/$ORCHESTRA_ROOT/revng" \
+          -v "$(readlink -f "$ORCHESTRA_DOTDIR/..")":/orchestra \
+          "$IMAGE_NAME" \
+          bash -c /orchestra/.orchestra/support/revng-distributable/test.sh;
+        then
+          echo "[!] $IMAGE_NAME test failed"
+          exit 1
+        else
+          echo "[+] $IMAGE_NAME test passed"
+        fi
+      done
+      echo "[+] revng-distributable tests passed"
+    fi
+  fi
 build_dependencies:
   - ui/cold-revng
 skip_post_install: true

.orchestra/config/components/ui/libX.yml

@@ -5,7 +5,7 @@
 #@ load("/lib/create_component.lib.yml", "single_build_component")
 #@ load("/lib/make.lib.yml", "make")
 
-#@ xcb_proto_source_url = "https://xorg.freedesktop.org/archive/individual/proto/xcb-proto-1.14.tar.gz"
+#@ xcb_proto_source_url = "https://xorg.freedesktop.org/archive/individual/proto/xcb-proto-1.14.1.tar.gz"
 #@ libxcb_source_url = "https://xorg.freedesktop.org/archive/individual/lib/libxcb-1.14.tar.gz"
 #@ libx11_source_url = "https://www.x.org/releases/individual/lib/libX11-1.6.12.tar.gz"
 #@ xtrans_source_url = "https://xorg.freedesktop.org/releases/individual/lib/xtrans-1.4.0.tar.bz2"

.orchestra/config/components/ui/libglvnd.yml

@@ -24,6 +24,9 @@ configure: |
     "$BUILD_DIR" \
     "$BUILD_DIR/source"
 build_system: ninja
+post_install: |
+  # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=881789
+  ln -s libGLX_mesa.so.0 "${TMP_ROOT}${ORCHESTRA_ROOT}/lib64/libGLX_indirect.so.0"
 build_dependencies:
   - host-c-toolchain
   - glibc

.orchestra/config/components/ui/qt.yml

@@ -61,6 +61,7 @@
     -no-glib \
     -no-cups \
     -no-gtk \
+    -qt-libmd4c \
     -qt-harfbuzz \
     -fontconfig \
     -system-freetype \

.orchestra/config/global_options.yml

@@ -36,3 +36,6 @@ branches:
   - develop
   - master
 nonredistributable_base_url: "https://dummydomain.com/nonredistributable/"
+
+#! see cold-revng component for the list of admissible test methods
+cold_revng_test_method: podman-xvfb

.orchestra/support/revng-distributable/dockers/README.md

@@ -0,0 +1,33 @@
+# Status of revng-distributable tests
+
+## How to run the tests
+
+To run revng-distributable tests, use `orc install --test revng-distributable`.
+By default, tests are run on all supported distros using podman and xvfb.
+The test mode can be changed by adding the following to `user_options.yml`:
+
+```yml
+#@overlay/replace
+cold_revng_test_method: podman-xvfb
+```
+
+See the supported test modes in `revng_distributable.yml`.
+
+## Known-working test modes
+
+The tests work on the following distros (podman+xvfb):
+
+- Archlinux
+- Centos {7,8}
+- Debian {7,8,9,10}
+- Fedora {29,30,31,32}
+- Opensuse 15.2
+- Ubuntu {16,18,20}.04
+- Voidlinux
+
+The tests also work running revng-ui directly on the host, using the running X11 server (`host-x11` test mode)
+
+## Broken distros
+
+- gentoo: need to revise the scripts which install dependencies
+- opensuse 15.{0,1}: zypper segfaults

.orchestra/support/revng-distributable/dockers/archlinux.docker

@@ -1,7 +1,7 @@
 FROM archlinux
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/broken/gentoo.docker

@@ -0,0 +1,12 @@
+FROM gentoo/stage3-amd64 AS base-gentoo
+
+COPY install-revng-dependencies /
+RUN echo 'FEATURES="$FEATURES -sandbox -usersandbox -ipc-sandbox -network-sandbox -pid-sandbox"' >> /etc/portage/make.conf
+RUN emerge-webrsync
+
+FROM base-gentoo
+RUN echo "sys-auth/elogind" >> "/etc/portage/package.mask"
+RUN /install-revng-dependencies --include-test-dependencies
+
+ENV DISPLAY=:0
+ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/opensuse-15.0.docker → .orchestra/support/revng-distributable/dockers/broken/opensuse-15.0.docker

@@ -1,7 +1,7 @@
 FROM opensuse/leap:15.0
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/opensuse-15.1.docker → .orchestra/support/revng-distributable/dockers/broken/opensuse-15.1.docker

@@ -1,7 +1,7 @@
 FROM opensuse/leap:15.1
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/build-all

@@ -3,8 +3,10 @@
 set -x
 set -e
 
-FILES=$(find -name "*.docker")
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+FILES=$(find "$DIR" -name "*.docker")
 
 for FILE in $FILES; do
-    ./build $(basename "${FILE%.*}")
+    FILE_PATH=$(realpath "$FILE")
+    "$DIR/build" "$(basename "${FILE%.*}")"
 done

.orchestra/support/revng-distributable/dockers/centos-7.docker

@@ -2,7 +2,7 @@ FROM centos:7
 
 COPY install-revng-dependencies /
 RUN echo '00000000000000000000000000000000' > /etc/machine-id && \
-    /install-revng-dependencies
+    /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/centos-8.docker

@@ -2,7 +2,7 @@ FROM centos:8
 
 COPY install-revng-dependencies /
 RUN echo '00000000000000000000000000000000' > /etc/machine-id && \
-    /install-revng-dependencies
+    /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/debian-10.docker

@@ -1,7 +1,7 @@
 FROM debian:10
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/debian-7.docker

@@ -3,7 +3,9 @@ FROM debian:7
 COPY install-revng-dependencies /
 RUN echo "deb http://archive.debian.org/debian/ wheezy contrib main non-free" > /etc/apt/sources.list && \
     echo "deb-src http://archive.debian.org/debian/ wheezy contrib main non-free" >> /etc/apt/sources.list && \
-    /install-revng-dependencies
+    echo "deb http://archive.debian.org/debian-security/ wheezy/updates contrib main non-free" >> /etc/apt/sources.list && \
+    echo "Acquire::Check-Valid-Until false;" >> /etc/apt/apt.conf && \
+    /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/debian-8.docker

@@ -1,9 +1,11 @@
 FROM debian:8
 
 COPY install-revng-dependencies /
-RUN echo "deb http://archive.debian.org/debian/ jessie contrib main non-free" > /etc/apt/sources.list && \
-    echo "deb-src http://archive.debian.org/debian/ jessie contrib main non-free" >> /etc/apt/sources.list && \
-    /install-revng-dependencies
+RUN echo "deb http://deb.debian.org/debian/ jessie contrib main non-free" > /etc/apt/sources.list && \
+    echo "deb-src http://deb.debian.org/debian/ jessie contrib main non-free" >> /etc/apt/sources.list && \
+    echo "deb http://deb.debian.org/debian-security/ jessie/updates contrib main non-free" >> /etc/apt/sources.list && \
+    echo "Acquire::Check-Valid-Until false;" >> /etc/apt/apt.conf && \
+    /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/debian-9.docker

@@ -1,7 +1,7 @@
 FROM debian:9
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/fedora-29.docker

@@ -1,7 +1,7 @@
 FROM fedora:29
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/fedora-30.docker

@@ -1,7 +1,7 @@
 FROM fedora:30
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/fedora-31.docker

@@ -1,7 +1,7 @@
 FROM fedora:31
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1

.orchestra/support/revng-distributable/dockers/fedora-32.docker

@@ -1,7 +1,7 @@
 FROM fedora:32
 
 COPY install-revng-dependencies /
-RUN /install-revng-dependencies
+RUN /install-revng-dependencies --include-test-dependencies
 
 ENV DISPLAY=:0
 ENV QT_DEBUG_PLUGINS=1