Merge branch 'self-hosted-ci'

Author: madadam

.github/workflows/ci.yml

@@ -16,30 +16,38 @@ env:
     ouisync/target/
 
 jobs:
-  test:
-    name: test on ${{ matrix.name }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        name: [linux, windows]
-        include:
-          - name: linux
-            os: ubuntu-24.04
-            env:
-              OUISYNC_LIB: ouisync/target/debug/libouisync_service.so
-            analyze: true
+  test_linux:
+    name: test on linux
+    runs-on: self-hosted
 
-          - name: windows
-            os: windows-latest
-            env:
-              # Install Dokan2.dll to where `flutter test` can find it
-              DOKAN_DLL_OUTPUT_PATH: C:\Windows\System32
-              OUISYNC_LIB: ouisync\target\debug\ouisync_service.dll
-            # FIXME: The mount test fails on windows CI with a panic somewhere in the dokan-rust
-            # crate. Disabling for now but it needs to be looked into.
-            test-args: --exclude-tags mount
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Create container name
+        run: echo "CONTAINER=ouisync-unit-test-linux-$GITHUB_RUN_ID" >> $GITHUB_ENV
+
+      - name: Start container
+        run: ./docker/linux.sh --container $CONTAINER --srcdir . --cache start
 
-    env: ${{ matrix.env }}
+      - name: Analyze
+        run: ./docker/linux.sh --container $CONTAINER analyze
+
+      - name: Run tests
+        run: ./docker/linux.sh --container $CONTAINER unit-test
+
+      - name: Stop container
+        run: ./docker/linux.sh --container $CONTAINER stop
+        if: always()
+
+  test_windows:
+    name: test on windows
+    runs-on: windows-latest
+    env:
+      # Install Dokan2.dll to where `flutter test` can find it
+      DOKAN_DLL_OUTPUT_PATH: C:\Windows\System32
+      OUISYNC_LIB: ouisync\target\debug\ouisync_service.dll
 
     steps:
       - uses: actions/checkout@v4
@@ -54,18 +62,10 @@ jobs:
           key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
 
-      - name: Install dependencies (linux)
-        run: |
-          sudo apt update -y
-          sudo apt-get install libfuse-dev
-        if: matrix.name == 'linux'
-
       # FIXME: when dokan is installed some tests fail with weird panic from the dokan-rust library.
-      # Disabling for now. See also the comment for the `test-args` variable in the `windows`
-      # strategy.
+      # Disabling for now. See also the comment in the 'Run tests' step.
       # - name: Install dependencies (windows)
       #   run: choco install dokany2
-      #   if: matrix.name == 'windows'
 
       - uses: subosito/flutter-action@v2
         with:
@@ -109,25 +109,25 @@ jobs:
       - name: "[HACK] Force dokan-sys to build"
         run: cargo clean --package dokan-sys
         working-directory: ouisync
-        if: matrix.name == 'windows'
 
       - name: Build Ouisync service library for tests
         working-directory: ouisync
         run: cargo build --package ouisync-service --lib
 
       - name: Run tests
-        run:
-          flutter test ${{ matrix.test-args }}
+        # FIXME: The mount test fails on windows CI with a panic somewhere in the dokan-rust
+        # crate. Disabling for now but it needs to be looked into.
+        run: flutter test --exclude-tags mount
 
       - name: Upload test/debug artifacts
         uses: actions/upload-artifact@v4
         with:
           # Name of the produced zip file
-          name: test-artifacts-${{ matrix.name }}
+          name: test-artifacts-${{ runner.os }}
           path: test/widget/artifacts/*
           if-no-files-found: ignore
-        # Upload even when tests fail
-        if: always()
+        # Upload when tests fail
+        if: failure()
 
   build:
     name: build on ${{ matrix.name }}
@@ -280,8 +280,7 @@ jobs:
 
   integration_test_android:
     name: "run integration tests on android ${{ matrix.api-level }}"
-    runs-on: ubuntu-24.04
-
+    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:
@@ -292,10 +291,7 @@ jobs:
           # - 25
 
           - 26
-
-          # This uses different target so it's included explicitly
-          # - 27
-
+          - 27
           - 28
           - 29
           - 30
@@ -305,50 +301,27 @@ jobs:
           - 34
           - 35
           - 36
-        target:
-          - google_apis
-        include:
-          - api-level: 27
-            target: default
-    steps:
-
-      - name: Enable KVM
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm
 
+    steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - uses: ./.github/actions/setup_android_tests
-
-      - name: Prebuild
-        run: flutter build apk --debug --flavor itest --target-platform android-x64
-
       - name: Run tests
-        uses: reactivecircus/android-emulator-runner@v2
-        with:
-          api-level: ${{ matrix.api-level }}
-          target: ${{ matrix.target }}
-          arch: x86_64
-          sdcard-path-or-size: 16M
-          script: |
-            adb logcat -c || true
-            adb logcat > logcat.txt &
-            yes | ./util/adb-format-sdcard.sh
-            flutter test integration_test --flavor itest --ignore-timeouts
-
-      - name: Upload logcat
-        uses: actions/upload-artifact@v4
-        with:
-          # Name of the produced zip file
-          name: logcat-${{ matrix.api-level }}
-          path: logcat.txt
-          if-no-files-found: ignore
-        # Upload only when the tests fail
-        if: failure() || cancelled()
+        run: |
+          ./docker/linux.sh \
+            --container ouisync-integration-test-android-${{ matrix.api-level }}-$GITHUB_RUN_ID \
+            integration-test --platform android --api ${{ matrix.api-level }}
+
+      # - name: Upload logcat
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     # Name of the produced zip file
+      #     name: logcat-${{ matrix.api-level }}
+      #     path: logcat.txt
+      #     if-no-files-found: ignore
+      #   # Upload only when the tests fail
+      #   if: failure() || cancelled()
 
 
   # TODO: Integration tests on both linux and windows currently fail or hang. Commenting them out until they are fixed.

docker/Dockerfile.build-linux docker/Dockerfile.linuxdocker/Dockerfile.build-linux renamed to docker/Dockerfile.linux

@@ -3,18 +3,22 @@
 FROM ubuntu:22.04
 
 ARG NDK_VERSION
+ARG RUST_VERSION=1.91.0
+ARG FLUTTER_VERSION=3.32.5
 
-ENV FLUTTER_VERSION=3.32.5
-ENV RUST_VERSION=1.89.0
-
-RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y \
-    build-essential cmake ninja-build clang pkg-config \
-    git xxd unzip \
-    wget curl rsync \
-    libgtk-3-dev libfuse-dev libappindicator3-dev libsecret-1-dev libcurl4-openssl-dev \
-    python3-minimal python-is-python3 \
-    openjdk-17-jdk
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+        build-essential cmake ninja-build clang pkg-config \
+        git xxd unzip \
+        wget curl rsync \
+        libgtk-3-dev libfuse-dev libappindicator3-dev libsecret-1-dev libcurl4-openssl-dev \
+        python3-minimal python-is-python3 \
+        openjdk-17-jdk \
+        libpulse0 \
+        fuse3 \
+        && \
+    apt-get clean
 
 # download and install sdkmanager
 # https://developer.android.com/studio#command-tools
@@ -30,19 +34,25 @@ RUN mkdir -p ${ANDROID_SDK_ROOT}/cmdline-tools && \
 ENV PATH="${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin:${PATH}"
 RUN yes | sdkmanager --licenses
 
-# These would be installed automatically, but good to have them in the image.
+# Install android packages (some of these would be installed automatically, but good to have them in
+# the image to speed things up).
+#
 # TODO: Why do we need different versions of `build-tools` and `platforms`?
-RUN sdkmanager --install \
-    "ndk;$NDK_VERSION" \
-    'build-tools;35.0.0' \
-    'build-tools;36.0.0' \
-    'platform-tools' \
-    'platforms;android-31' \
-    'platforms;android-33' \
-    'platforms;android-34' \
-    'platforms;android-35' \
-    'cmake;3.22.1' \
-    'cmdline-tools;latest'
+RUN sdkmanager --install    \
+    "cmdline-tools;latest"  \
+    "build-tools;35.0.0"    \
+    "build-tools;36.0.0"    \
+    "platform-tools"        \
+    "platforms;android-31"  \
+    "platforms;android-33"  \
+    "platforms;android-34"  \
+    "platforms;android-35"  \
+    "platforms;android-36"  \
+    "emulator"              \
+    "ndk;$NDK_VERSION"      \
+    "cmake;3.22.1"
+
+ENV PATH="${ANDROID_SDK_ROOT}/platform-tools:${ANDROID_SDK_ROOT}/emulator:${PATH}"
 
 # install flutter
 RUN wget https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_${FLUTTER_VERSION}-stable.tar.xz -P /tmp
@@ -56,19 +66,27 @@ RUN git config --global --add safe.directory /opt/flutter
 
 RUN flutter config --enable-android --enable-linux-desktop
 RUN flutter precache --linux
-RUN flutter config --no-analytics
+RUN flutter config --no-analytics --no-cli-animations
 RUN yes | flutter doctor --android-licenses
+RUN dart --disable-analytics
 RUN flutter doctor -v
 
-# install rust
+# Install rust
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=${RUST_VERSION} -y
 ENV PATH="/root/.cargo/bin:$PATH"
 RUN rustup target add aarch64-linux-android armv7-linux-androideabi x86_64-linux-android
 
+# Install some rust utilities
+RUN cargo install cargo-sweep
+
 # Expose Java
 # Sentry started to require JDK even for non Android builds. This requirement
 # might be removed in the future.
 # https://github.com/getsentry/sentry-dart/issues/3033#issuecomment-3140320200
 ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
 
+ENV LANG="C.UTF-8"
+ENV LANGUAGE="C.UTF-8"
+ENV LC_ALL="C.UTF-8"
+
 WORKDIR /opt

docker/README.md

@@ -1,19 +1,16 @@
-# Docker building
+# Testing and building in Docker
 
-This folder contains scripts for building Ouisync App in remote docker containers.
+This folder contains scripts for testing and building Ouisync App in docker containers.
 
-The `build-linux.sh` script will build `aab`, `apk`, `deb-cli` and `deb-gui` packages.
-The `build-windows.sh` script will build `msix` and `exe` packages.
+- The `linux.sh` script is used to run unit and integration tests and to build release packages for
+  linux and android.
+- The `build-windows.sh` script is used to build release packages for windows.
 
 ## Usage
 
-Assuming you have machines running Docker listed in your `~/.ssh/config` as `linux_machine` and `windows_machine`, run
-
-```bash
-./docker/build-linux.sh --host linux_machine --commit <COMMIT_HASH>
-./docker/build-windows.sh --host windows_machine --commit <COMMIT_HASH>
-```
+Run `./docker/linux.sh --help` and `./docker/build-windows.sh --help` for usage instructions.
 
 ## Requirements
 
-Have [pass](https://www.passwordstore.org/) installed with eQualitie's password repository.
+To build the release packages, have [pass](https://www.passwordstore.org/) installed with
+eQualitie's password repository.