databend-performance-dispatch.yaml

on:
  repository_dispatch:
    types: [ run_perf, rerun_perf, rerun_perf_all ]
name: Build Docker Image on official docker hub (primary for benchmarking and CI tests)
jobs:
  slack-notification-start:
    name: Slack Notification Start
    runs-on: ubuntu-latest
    steps:
      - uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_ICON: https://avatars.githubusercontent.com/u/80994548?s=200&v=4
          SLACK_COLOR: good
          SLACK_CHANNEL: github-activity
          SLACK_MESSAGE: 'start databend performance test!'
          SLACK_TITLE: databend performance test
          SLACK_USERNAME: databend messager
  build-current:
    name: Build current databend image
    if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' || github.event.action == 'rerun_perf' }}
    runs-on: perf-test
    env:
      GITHUB_TOKEN: ${{ github.token }}
      GITHUB_ORG: datafuselabs
      GITHUB_REPO: databend
      GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      SHA: ${{ github.event.client_payload.CURRENT_BRANCH }}
      PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
      LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
      UUID: ${{ github.event.client_payload.UUID }}
      ITERATION: 100
      REGION: ${{ github.event.client_payload.REGION }}
      BUCKET: ${{ github.event.client_payload.BUCKET }}
      ENDPOINT: ${{ github.event.client_payload.ENDPOINT }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: us-west-2
      AWS_DEFAULT_OUTPUT: json
    steps:
      - name: Update status to pending
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        run: |
          echo "update github pr status"
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"pending","context":"build-current-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Checkout given branch
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: actions/checkout@v2
        with:
          ref: ${{ env.SHA }}
      - name: build all binary
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        id: make_release
        run: |
          # enable sccache
          sccache --start-server
          echo $HOME
          export SCCACHE_DIR=/home/runner/.cache/sccache
          export RUSTC_WRAPPER=sccache
          cargo build --bin=databend-query --bin=databend-meta --target x86_64-unknown-linux-gnu --release
          mkdir -p distro
          sccache --show-stats
          mv target/x86_64-unknown-linux-gnu/release/databend-query ./distro/databend-query
          mv target/x86_64-unknown-linux-gnu/release/databend-meta ./distro/databend-meta
          mv docker/bootstrap.sh ./distro/bootstrap.sh
          mv docker/databend-query-docker.toml ./distro/databend-query-docker.toml
      - name: Set up QEMU
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        id: buildx
        uses: docker/setup-buildx-action@v1
      - name: Login to DockerHub
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        id: docker_build
        uses: docker/build-push-action@v2
        with:
          push: true
          tags: ${{ secrets.DOCKERHUB_NAMESPACE }}/databend:perf-${{ env.SHA }}
          context: .
          platforms: linux/amd64
          file: ./docker/cache/Dockerfile
      - name: bypass
        if: ${{  github.event.action == 'rerun_perf' }}
        run: |
          echo "bypass build current image step"
        #    - name: Delete all resources
      #      if: always()
      #      run: |
      #        sccache --stop-server
      - name: Update status to failure
        if: failure()
        run: |
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"failure","context":"build-current-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Update status to success
        if: success()
        run: |
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"success","context":"build-current-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
  build-release:
    name: Build reference databend image
    if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' || github.event.action == 'rerun_perf' }}
    runs-on: perf-test
    env:
      GITHUB_TOKEN: ${{ github.token }}
      GITHUB_ORG: datafuselabs
      GITHUB_REPO: databend
      GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      SHA: ${{ github.event.client_payload.REF_BRANCH }}
      PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
      LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
      UUID: ${{ github.event.client_payload.UUID }}
      ITERATION: 10
      REGION: ${{ github.event.client_payload.REGION }}
      BUCKET: ${{ github.event.client_payload.BUCKET }}
      ENDPOINT: ${{ github.event.client_payload.ENDPOINT }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: us-west-2
      AWS_DEFAULT_OUTPUT: json
    steps:
      - name: Update status to pending
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        run: |
          echo "update github pr status"
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"pending","context":"build-ref-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Check out branch
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: actions/checkout@v2
        with:
          ref: ${{ env.SHA }}
      - name: build all binary
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        id: make_debug
        run: |
          sccache --start-server
          echo $HOME
          export SCCACHE_DIR=/home/runner/.cache/sccache
          export RUSTC_WRAPPER=sccache
          cargo build --bin=databend-query --bin=databend-meta --target x86_64-unknown-linux-gnu --release
          mkdir -p distro
          sccache --show-stats
          mv target/x86_64-unknown-linux-gnu/release/databend-query ./distro/databend-query
          mv target/x86_64-unknown-linux-gnu/release/databend-meta ./distro/databend-meta
          mv docker/bootstrap.sh ./distro/bootstrap.sh
      - name: Set up QEMU
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        id: buildx
        uses: docker/setup-buildx-action@v1
      - name: Login to DockerHub
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        id: docker_build
        if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' }}
        uses: docker/build-push-action@v2
        with:
          push: true
          tags: ${{ secrets.DOCKERHUB_NAMESPACE }}/databend:perf-${{ env.SHA }}
          context: .
          platforms: linux/amd64
          file: ./docker/cache/Dockerfile
      - name: bypass
        if: ${{  github.event.action == 'rerun_perf' }}
        run: |
          echo "bypass build ref image step"

      - name: Update status to failure
        if: failure()
        run: |
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"failure","context":"build-ref-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Update status to success
        if: success()
        run: |
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"success","context":"build-ref-image-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
  run_funcbench:
    name: Running databend single performance testing
    if: ${{ github.event.action == 'run_perf' || github.event.action == 'rerun_perf_all' || github.event.action == 'rerun_perf' }}
    runs-on: perf-test
    needs: [ build-current, build-release ]
    env:
      GITHUB_TOKEN: ${{ github.token }}
      GITHUB_ORG: datafuselabs
      GITHUB_REPO: databend
      GITHUB_STATUS_TARGET_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      CPU: 4300m
      MEMORY: 4Gi
      # define reference and compare branches for performance test
      # suppose branch building for current and reference works well
      CURRENT: ${{ github.event.client_payload.CURRENT_BRANCH }}
      REFERENCE: ${{ github.event.client_payload.REF_BRANCH }}
      PR_NUMBER: ${{ github.event.client_payload.PR_NUMBER }}
      LAST_COMMIT_SHA: ${{ github.event.client_payload.LAST_COMMIT_SHA }}
      UUID: ${{ github.event.client_payload.UUID }}
      ITERATION: 10
      # S3 storage for report and status updates
      CLUSTER_NAME: ${{ github.run_id }}
      REGION: ${{ github.event.client_payload.REGION }}
      BUCKET: ${{ github.event.client_payload.BUCKET }}
      ENDPOINT: ${{ github.event.client_payload.ENDPOINT }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: us-west-2
      AWS_DEFAULT_OUTPUT: json
      DELETE_CLUSTER_AFTER_RUN: false
    steps:
      - name: Update status to pending
        run: |
          echo "update github pr status"
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"pending","context":"run-perf-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"

      - name: Checkout tools repo
        uses: actions/checkout@v2
        with:
          repository: datafuselabs/test-infra
          ref: main
      - name: build latest binary
        run: |
          export PATH=$PATH:/usr/local/go/bin
          make build
      - uses: AbsaOSS/k3d-action@v1.5.0
        name: "Create Single Cluster"
        with:
          cluster-name: ${{env.CLUSTER_NAME}}
          args: >-
            -p "8083:80@agent[0]"
            -p "8443:443@agent[0]"
            -p "5053:53/udp@agent[0]"
            -p "8084:80@agent[1]"
            -p "8444:443@agent[1]"
            -p "5054:53/udp@agent[1]"
            --api-port 127.0.0.1:6445
            --agents 2
            --no-lb
            --k3s-server-arg "--no-deploy=traefik,servicelb,metrics-server"
            --timeout 360s
            --wait
      - name: testing environment set up
        run: |
          kubectl label node k3d-${{ env.CLUSTER_NAME }}-agent-0 test-branch=${{env.CURRENT}}
          kubectl label node k3d-${{ env.CLUSTER_NAME }}-agent-1 test-branch=${{env.REFERENCE}}
          kubectl wait -n kube-system --timeout=360s --for condition=Ready --all pods
          kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/namespace.yaml
          kubectl apply -f ./manifests/lb_configs.yaml
          kubectl delete ns runner-system --ignore-not-found
          kubectl create ns runner-system
          kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/metallb.yaml
          kubectl wait -n metallb-system --timeout=360s --for condition=Ready --all pods

      - name: Start performance testing
        id: perf
        run: >-
          echo ${{ env.UUID }};
          export ENDPOINT=$(echo $ENDPOINT | grep -oP '(http|https)://\K\S+');
          echo $ENDPOINT;
          make resource_apply -j 2;
          echo "Successfully deployed test pods, start to do performance test";
          make run_perf -j 2;
          kubectl logs -n default -l tag=current -f >current.log &
          kubectl logs -n default -l tag=ref -f >ref.log &
          kubectl wait --for=condition=complete jobs --all -n default --timeout=7200s 2>/dev/null;
          echo "Finished performance tests, upload reports";
          aws s3 --endpoint-url ${{ env.ENDPOINT }} cp ./current.log s3://$BUCKET/${{ env.PR_NUMBER}}/${{env.LAST_COMMIT_SHA}}/${{env.UUID}}/log/current.log;
          aws s3 --endpoint-url ${{ env.ENDPOINT }} cp ./ref.log s3://$BUCKET/${{ env.PR_NUMBER}}/${{env.LAST_COMMIT_SHA}}/${{env.UUID}}/log/ref.log;
          make run_compare;
          kubectl wait --for=condition=complete jobs --all -n default --timeout=7200s 2>/dev/null;
          export REPORT=${{ env.ENDPOINT }}/report/${{ env.PR_NUMBER}}/${{env.LAST_COMMIT_SHA}}/${{env.UUID}}/performance.html;
          export UPLOADED=$(for i in {1..10}; do if [[ $(curl -i -s  $REPORT | grep -c "<Code>NoSuchKey</Code>") == 0 ]]; then echo "Uploaded" && break; else kubectl delete jobs -n default --all && make run_compare && kubectl wait --for=condition=complete jobs --all -n default --timeout=7200s 2>/dev/null && echo "retry"; fi; done | grep -c "Uploaded");
          if [[ $UPLOADED == 0 ]]; then echo "cannot upload compare file to S3" && exit 1; else echo "upload success!"; fi;
          echo ::set-output name=report::"report url: $REPORT"
      - name: Update status to failure
        if: failure()
        run: |
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"failure","context":"run-perf-status","target_url":"'$GITHUB_STATUS_TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Slack Notification Fail
        if: failure()
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_ICON: https://avatars.githubusercontent.com/u/80994548?s=200&v=4
          SLACK_COLOR: ${{ job.status }}
          SLACK_CHANNEL: github-activity
          SLACK_MESSAGE: ${{ env.GITHUB_STATUS_TARGET_URL }}
          SLACK_TITLE: databend performance test
          SLACK_USERNAME: databend messager
      - name: Update status to success
        if: success()
        run: |
          export TARGET_URL=${{env.ENDPOINT}}/report/${{ env.PR_NUMBER }}/${{env.LAST_COMMIT_SHA}}/${{ env.UUID }}/performance.html
          curl -i -X POST \
          -H "Authorization: Bearer $GITHUB_TOKEN" \
          -H "Content-Type: application/json" \
          --data '{"state":"success","context":"run-perf-status","target_url":"'$TARGET_URL'"}' \
          "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/$LAST_COMMIT_SHA"
      - name: Slack Notification Success
        if: success()
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_ICON: https://avatars.githubusercontent.com/u/80994548?s=200&v=4
          SLACK_COLOR: ${{ job.status }}
          SLACK_CHANNEL: github-activity
          SLACK_MESSAGE: ${{ steps.perf.outputs.report }}
          SLACK_TITLE: databend performance test
          SLACK_USERNAME: databend messager
      - name: Delete all resources
        if: always()
        run: |
          k3d cluster delete ${{ env.CLUSTER_NAME }}