build_test.yml

name: test

on:
  push:
    branches:    
      - master
      - 'release-**'
  pull_request:
    branches:    
      - master
      - 'release-**'

jobs:
  unit_test:
    strategy:
      matrix:
        os: [ubuntu-latest, ubuntu-24.04-arm]
    name: unit tests
    runs-on: ${{ matrix.os }}

    services:
      mysql:
        image: mysql:8.0
        ports:
          - 3306
        env:
          MYSQL_ROOT_PASSWORD: password
        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3

      postgres:
        image: postgres:10.0
        ports:
          - 5432
        env:
          POSTGRES_USER: gorse
          POSTGRES_PASSWORD: gorse_pass
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      mongo:
        image: mongo:6
        ports:
          - 27017
        env:
          MONGO_INITDB_ROOT_USERNAME: root
          MONGO_INITDB_ROOT_PASSWORD: password
        options: >-
          --health-cmd mongosh
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      clickhouse:
        image: clickhouse/clickhouse-server:26.2
        ports:
          - 8123
        env:
          CLICKHOUSE_USER: gorse
          CLICKHOUSE_PASSWORD: gorse_pass
        options: >-
          --health-cmd="clickhouse-client --query 'SELECT 1'" 
          --health-interval=10s 
          --health-timeout=5s 
          --health-retries=5

      redis:
        image: redis:8
        ports:
          - 6379

      rustfs:
        image: rustfs/rustfs:alpha
        ports:
          - 9000
        env:
          RUSTFS_ACCESS_KEY: rustfsadmin
          RUSTFS_SECRET_KEY: rustfsadmin

      qdrant:
        image: qdrant/qdrant:latest
        ports:
          - 6334
      
      weaviate:
        image: cr.weaviate.io/semitechnologies/weaviate:1.35.7
        ports:
          - 8080

    steps:
    - name: Set up dataset
      run: |
        mkdir -p ~/.gorse/dataset
        mkdir -p ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/mnist.zip -P ~/.gorse/download
        unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/mnist.zip -d ~/.gorse/dataset

    - uses: actions/checkout@v2
    - uses: actions/setup-go@v5
      with:
        go-version-file: ./go.mod

    - name: Install and start Azurite
      uses: potatoqualitee/azuright@v1

    - name: Setup Milvus
      run: |
        curl -sfL https://raw.githubusercontent.com/milvus-io/milvus/6da09b8b02b1/scripts/standalone_embed.sh -o standalone_embed.sh
        bash standalone_embed.sh start
      working-directory: ${{ runner.temp }}

    - name: Test
      run: go test -timeout 30m ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...
      env:
        # MySQL
        MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mysql.ports[3306] }})/
        # Postgres
        POSTGRES_URI: postgres://gorse:gorse_pass@localhost:${{ job.services.postgres.ports[5432] }}/
        # MongoDB
        MONGO_URI: mongodb://root:password@localhost:${{ job.services.mongo.ports[27017] }}/
        # ClickHouse
        CLICKHOUSE_URI: clickhouse://gorse:gorse_pass@localhost:${{ job.services.clickhouse.ports[8123] }}/
        # Redis
        REDIS_URI: redis://localhost:${{ job.services.redis.ports[6379] }}/
        # S3
        S3_ENDPOINT: localhost:${{ job.services.rustfs.ports[9000] }}
        S3_ACCESS_KEY_ID: rustfsadmin
        S3_SECRET_ACCESS_KEY: rustfsadmin
        # Azure Blob (Azurite)
        AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
        # Qdrant
        QDRANT_URI: qdrant://localhost:${{ job.services.qdrant.ports[6334] }}/
        # Weaviate
        WEAVIATE_URI: weaviate://localhost:${{ job.services.weaviate.ports[8080] }}/

    - name: Upload coverage to Codecov
      if: matrix.os == 'ubuntu-latest'
      uses: codecov/codecov-action@v5
      with:
        files: ./coverage.txt
        fail_ci_if_error: false
        token: ${{ secrets.CODECOV_TOKEN }}

  unit_test_macos:
    name: unit tests (macos-latest)
    runs-on: macos-latest
    steps:
    - name: Set up dataset
      run: |
        mkdir -p ~/.gorse/dataset
        mkdir -p ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download
        wget https://cdn.gorse.io/datasets/mnist.zip -P ~/.gorse/download
        unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset
        unzip ~/.gorse/download/mnist.zip -d ~/.gorse/dataset

    - uses: actions/checkout@v2
    - uses: actions/setup-go@v5
      with:
        go-version-file: ./go.mod

    - name: Test
      run: go test -timeout 20m ./...

  unit_test_windows:
    strategy:
      matrix:
        os: [windows-latest, windows-11-arm]
    name: unit tests
    runs-on: ${{ matrix.os }}
    steps:
    - name: Set up dataset
      run: |
        New-Item -Type Directory -Path ~/.gorse/dataset
        New-Item -Type Directory -Path ~/.gorse/download
        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-100k.zip -OutFile ~/.gorse/download/ml-100k.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-1m.zip -OutFile ~/.gorse/download/ml-1m.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/pinterest-20.zip -OutFile ~/.gorse/download/pinterest-20.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/frappe.zip -OutFile ~/.gorse/download/frappe.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-tag.zip -OutFile ~/.gorse/download/ml-tag.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/criteo.zip -OutFile ~/.gorse/download/criteo.zip
        Invoke-WebRequest https://cdn.gorse.io/datasets/mnist.zip -OutFile ~/.gorse/download/mnist.zip
        Expand-Archive ~/.gorse/download/ml-100k.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/ml-1m.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/pinterest-20.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/frappe.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/ml-tag.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/criteo.zip -DestinationPath ~/.gorse/dataset
        Expand-Archive ~/.gorse/download/mnist.zip -DestinationPath ~/.gorse/dataset

    - uses: actions/checkout@v2
    - uses: actions/setup-go@v5
      with:
        go-version-file: ./go.mod

    - name: Test
      run: go test -timeout 20m ./...

  integrate_test:
    name: integrate tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        database: [mysql, postgres, mongo, sqlite]
    steps:
    - uses: actions/checkout@v5
      with:
        fetch-depth: 0

    - uses: actions/setup-go@v5
      with:
        go-version-file: ./go.mod

    - uses: cuchi/jinja2-action@v1.2.0
      with:
        template: client/docker-compose.yml.j2
        output_file: docker-compose.yml
        strict: true
        variables: |
          database=${{ matrix.database }}

    - name: Setup test
      run: ./client/setup-test.sh
      env:
        DOCKER_BUILDKIT: 1

    - name: Test
      run: go test ./client/
      env:
        GORSE_SERVER_ENDPOINT: http://localhost:8087
        GORSE_DASHBOARD_ENDPOINT: http://localhost:8088

  compat_test:
    name: compatibility tests
    runs-on: ubuntu-latest

    services:
      mariadb:
        image: mariadb:10.2
        ports:
          - 3306
        env:
          MYSQL_ROOT_PASSWORD: password

      kvrocks:
        image: apache/kvrocks:nightly
        ports:
          - 6666

    steps:
      - name: Install pre-requisites
        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: qemu-user-static

      - name: Setup Redis cluster
        uses: gorse-cloud/redis-cluster@v1.0.0

      - uses: actions/checkout@v2
      - uses: actions/setup-go@v5
        with:
          go-version-file: ./go.mod

      - name: Test RISC-V floats
        run: make -C common/floats test-rvv
        env:
          RISCV64_QEMU: qemu-riscv64-static

      - name: Test LoongArch LASX floats
        run: make -C common/floats test-lasx
        env:
          LOONG64_QEMU: qemu-loongarch64-static

      - name: Test MariaDB
        run: go test ./storage/data ./storage/cache -run TestMySQL
        env:
          MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mariadb.ports[3306] }})/

      - name: Test Kvrocks
        run: go test ./storage/cache -run ^TestRedis
        env:
          REDIS_URI: redis://localhost:${{ job.services.kvrocks.ports[6666] }}/

      - name: Test Redis cluster (1 address)
        run: go test ./storage/cache -run ^TestRedis
        env:
          REDIS_URI: redis+cluster://localhost:7000
      
      - name: Test Redis cluster (6 addresses)
        run: go test ./storage/cache -run ^TestRedis
        env:
          REDIS_URI: redis+cluster://localhost:7000?addr=localhost:7001&addr=localhost:7002&addr=localhost:7003&addr=localhost:7004&addr=localhost:7005

  playground:
    name: playground
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Build Docker image
        run: docker build -t zhenghaoz/gorse-in-one -f ./cmd/gorse-in-one/Dockerfile .
      - name: Run playground
        run: docker run -d -p 8088:8088 --name playground zhenghaoz/gorse-in-one --playground
      - name: Check dashboard URL
        run: |
          for i in {1..10}; do
            curl -sSf http://localhost:8088 && break
            docker logs playground
            sleep 10
          done

  golangci:
    name: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-go@v5
        with:
          go-version-file: ./go.mod
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v9
        with:
          args: --timeout 20m