Skip to content

Commit 8b5e9cb

Browse files
zhouyuanLakehouse Engine Bot
zhouyuan
authored and
Lakehouse Engine Bot
committed
adding daily tests
Signed-off-by: Yuan <[email protected]> Alchemy-item: [fix: Adding daily tests](#44 (comment)) commit 1/1 - 25dda9d
1 parent 77e4c12 commit 8b5e9cb

File tree

2 files changed

+176
-15
lines changed

2 files changed

+176
-15
lines changed

‎.github/workflows/gluten_daily.yml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Gluten Daily Build
16+
17+
on:
18+
schedule:
19+
- cron: '0 0 * * *'
20+
env:
21+
MVN_CMD: 'mvn -ntp'
22+
23+
jobs:
24+
25+
gluten-cpp-build:
26+
name: gluten cpp build
27+
# prevent errors when forks ff their main branch
28+
if: ${{ github.repository == 'IBM/velox' }}
29+
runs-on: ubuntu-22.04
30+
env:
31+
CCACHE_DIR: "${{ github.workspace }}/.ccache"
32+
steps:
33+
- uses: actions/checkout@v4
34+
- name: Get Ccache
35+
uses: actions/cache/restore@v4
36+
with:
37+
path: '${{ env.CCACHE_DIR }}'
38+
key: ccache-centos7-release-default-${{github.sha}}
39+
- name: Setup Gluten
40+
run: |
41+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
42+
BRANCH=$(echo ${GITHUB_REF#refs/heads/})
43+
sed -i 's/oap-project/IBM/g' ep/build-velox/src/get_velox.sh
44+
sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=main/g' ep/build-velox/src/get_velox.sh
45+
- name: Build Gluten native libraries
46+
run: |
47+
docker pull apache/gluten:vcpkg-centos-7
48+
docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
49+
set -e
50+
df -a
51+
cd /work/gluten
52+
export CCACHE_DIR=/work/.ccache
53+
mkdir -p /work/.ccache
54+
bash dev/ci-velox-buildstatic-centos-7.sh
55+
ccache -s
56+
mkdir -p /work/.m2/repository/org/apache/arrow/
57+
cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
58+
"
59+
- name: "Save ccache"
60+
uses: actions/cache/save@v4
61+
id: ccache
62+
with:
63+
path: '${{ env.CCACHE_DIR }}'
64+
key: ccache-centos7-release-default-${{github.sha}}
65+
- uses: actions/upload-artifact@v4
66+
with:
67+
name: velox-native-lib-centos-7-${{github.sha}}
68+
path: ./gluten/cpp/build/releases/
69+
if-no-files-found: error
70+
- uses: actions/upload-artifact@v4
71+
with:
72+
name: arrow-jars-centos-7-${{github.sha}}
73+
path: .m2/repository/org/apache/arrow/
74+
if-no-files-found: error
75+
76+
linux-gcc:
77+
name: Build with GCC
78+
if: ${{ github.repository == 'IBM/velox' }}
79+
runs-on: ubuntu-22.04
80+
container: ghcr.io/facebookincubator/velox-dev:adapters
81+
defaults:
82+
run:
83+
shell: bash
84+
env:
85+
CCACHE_DIR: ${{ github.workspace }}/ccache
86+
VELOX_DEPENDENCY_SOURCE: SYSTEM
87+
GTest_SOURCE: BUNDLED
88+
cudf_SOURCE: BUNDLED
89+
CUDA_VERSION: '12.8'
90+
faiss_SOURCE: BUNDLED
91+
steps:
92+
- uses: actions/checkout@v4
93+
with:
94+
fetch-depth: 2
95+
persist-credentials: false
96+
97+
- name: Fix git permissions
98+
# Usually actions/checkout does this but as we run in a container
99+
# it doesn't work
100+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
101+
102+
- name: Install Dependencies
103+
run: |
104+
if git diff --name-only HEAD^1 HEAD | grep -q "scripts/setup-"; then
105+
# Overwrite old setup scripts with changed versions
106+
cp scripts/setup-* /
107+
108+
mkdir /tmp/build
109+
cd /tmp/build
110+
source /opt/rh/gcc-toolset-12/enable
111+
# install basic deps
112+
bash /setup-centos9.sh
113+
114+
source /setup-centos9.sh
115+
install_adapters
116+
install_cuda $CUDA_VERSION
117+
118+
cd /
119+
rm -rf /tmp/build # cleanup to avoid issues with disk space
120+
fi
121+
122+
- name: Install Minio
123+
run: |
124+
MINIO_BINARY="minio-2022-05-26"
125+
if [ ! -f /usr/local/bin/${MINIO_BINARY} ]; then
126+
wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z -O ${MINIO_BINARY}
127+
chmod +x ./${MINIO_BINARY}
128+
mv ./${MINIO_BINARY} /usr/local/bin/
129+
fi
130+
131+
- uses: actions/cache/restore@v4
132+
with:
133+
path: '${{ env.CCACHE_DIR }}'
134+
key: ccache-linux-adapters-${{ inputs.use-clang && 'clang' || 'gcc' }}-${{github.sha}}
135+
restore-keys: |
136+
ccache-linux-adapters-${{ inputs.use-clang && 'clang' || 'gcc' }}
137+
138+
- name: Zero Ccache Statistics
139+
run: |
140+
ccache -sz
141+
142+
- name: Make Release Build
143+
env:
144+
#MAKEFLAGS: 'NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4'
145+
CUDA_ARCHITECTURES: 70
146+
CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc
147+
# Set compiler to GCC 12
148+
CUDA_FLAGS: -ccbin /opt/rh/gcc-toolset-12/root/usr/bin
149+
run: |
150+
EXTRA_CMAKE_FLAGS=(
151+
"-DVELOX_ENABLE_BENCHMARKS=ON"
152+
"-DVELOX_ENABLE_EXAMPLES=ON"
153+
"-DVELOX_ENABLE_ARROW=ON"
154+
"-DVELOX_ENABLE_GEO=ON"
155+
"-DVELOX_ENABLE_FAISS=ON"
156+
"-DVELOX_ENABLE_PARQUET=ON"
157+
"-DVELOX_ENABLE_HDFS=ON"
158+
"-DVELOX_ENABLE_S3=ON"
159+
"-DVELOX_ENABLE_GCS=ON"
160+
"-DVELOX_ENABLE_ABFS=ON"
161+
"-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
162+
"-DVELOX_ENABLE_CUDF=ON"
163+
"-DVELOX_ENABLE_WAVE=ON"
164+
"-DVELOX_MONO_LIBRARY=ON"
165+
"-DVELOX_BUILD_SHARED=ON"
166+
)
167+
if [[ "${USE_CLANG}" = "true" ]]; then scripts/setup-centos9.sh install_clang15; export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; CUDA_FLAGS="-ccbin /usr/lib64/llvm15/bin/clang++-15"; fi
168+
make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
169+
170+
- name: Ccache after
171+
run: ccache -s
172+
173+
- uses: actions/cache/save@v4
174+
with:
175+
path: '${{ env.CCACHE_DIR }}'
176+
key: ccache-linux-adapters-gcc-${{github.sha}}

‎.github/workflows/linux-build.yml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,6 @@
1515
name: Linux Build using GCC
1616

1717
on:
18-
push:
19-
branches:
20-
- main
21-
paths:
22-
- velox/**
23-
- '!velox/docs/**'
24-
- CMakeLists.txt
25-
- CMake/**
26-
- scripts/setup-ubuntu.sh
27-
- scripts/setup-common.sh
28-
- scripts/setup-versions.sh
29-
- scripts/setup-helper-functions.sh
30-
- .github/workflows/linux-build.yml
31-
- .github/workflows/linux-build-base.yml
32-
3318
pull_request:
3419
paths:
3520
- velox/**

0 commit comments

Comments
 (0)