Skip to content

Commit f2b3e30

Browse files
diveartdiveart
authored
Merge branch 'master-databricks' into databricks-patch-1
2 parents c36c28d + 59eb145 commit f2b3e30

36 files changed

+1145
-28
lines changed

‎.github/workflows/apply_datafold_monitors_config.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Apply monitors.yaml configuration to Datafold app
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
paths:
8+
- 'monitors.yaml'
9+
10+
jobs:
11+
apply:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: Set up Python
16+
uses: actions/setup-python@v2
17+
with:
18+
python-version: 3.12
19+
- name: Install dependencies
20+
run: |
21+
echo "setuptools<72" > constraints.txt
22+
export PIP_CONSTRAINT=constraints.txt
23+
pip install --upgrade pip
24+
python -m pip install --upgrade pip
25+
pip install datafold-sdk
26+
- name: Run apply.py
27+
run: |
28+
datafold monitors provision monitors.yaml
29+
env:
30+
DATAFOLD_HOST: https://app.datafold.com
31+
DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }}

‎.github/workflows/branch_replication.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ jobs:
3131
# BigQuery
3232
git branch -f master-bigquery master
3333
git push -u origin master-bigquery --force
34+
# Dremio
35+
git branch -f master-dremio master
36+
git push -u origin master-dremio --force
3437
3538
3639
- name: Verify move

‎.github/workflows/datagen.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ jobs:
2020

2121
- name: Install dependencies
2222
run: |
23+
echo "setuptools<72" > constraints.txt
24+
export PIP_CONSTRAINT=constraints.txt
2325
python -m pip install --upgrade pip
2426
pip install -r datagen/requirements.txt
2527
@@ -53,6 +55,7 @@ jobs:
5355
5456
- name: Commit new files
5557
run: |
58+
rm constraints.txt
5659
git add -A
5760
git commit -m "New data obtained on $(date)"
5861

‎.github/workflows/pr_job.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@ jobs:
2727
python-version: '3.8'
2828

2929
- name: install requirements
30-
run: pip install -q -r requirements.txt
30+
run: |
31+
echo "setuptools<72" > constraints.txt
32+
export PIP_CONSTRAINT=constraints.txt
33+
pip install --upgrade pip
34+
pip install -q -r requirements.txt
3135
3236
- name: install datafold-sdk
3337
run: pip install -q datafold-sdk

‎.github/workflows/pr_job_bigquery.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ jobs:
2626
python-version: '3.8'
2727

2828
- name: install requirements
29-
run: pip install -q -r requirements.txt
29+
run: |
30+
echo "setuptools<72" > constraints.txt
31+
export PIP_CONSTRAINT=constraints.txt
32+
pip install --upgrade pip
33+
pip install -q -r requirements.txt
3034
3135
- name: install datafold-sdk
3236
run: pip install -q datafold-sdk

‎.github/workflows/pr_job_databricks.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ jobs:
2626
python-version: '3.8'
2727

2828
- name: install requirements
29-
run: pip install -q -r requirements.txt
29+
run: |
30+
echo "setuptools<72" > constraints.txt
31+
export PIP_CONSTRAINT=constraints.txt
32+
pip install --upgrade pip
33+
pip install -q -r requirements.txt
3034
3135
- name: install datafold-sdk
3236
run: pip install -q datafold-sdk

‎.github/workflows/pr_job_dremio.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: dbt PR job (Dremio)
2+
3+
on:
4+
pull_request:
5+
types:
6+
- opened
7+
- reopened
8+
- synchronize
9+
- ready_for_review
10+
branches:
11+
- master-dremio
12+
# push:
13+
# branches-ignore:
14+
# - master
15+
# - master-dremio
16+
17+
jobs:
18+
run:
19+
runs-on: ubuntu-20.04
20+
steps:
21+
- name: checkout
22+
uses: actions/checkout@v2
23+
24+
- uses: actions/setup-python@v2
25+
with:
26+
python-version: '3.8'
27+
28+
- name: install requirements
29+
run: |
30+
echo "setuptools<72" > constraints.txt
31+
export PIP_CONSTRAINT=constraints.txt
32+
pip install --upgrade pip
33+
pip install -q -r requirements.txt
34+
35+
- name: install datafold-sdk
36+
run: pip install -q datafold-sdk
37+
38+
- name: dbt deps
39+
run: dbt deps
40+
41+
- name: Find Current Pull Request
42+
uses: jwalton/[email protected]
43+
id: findPR
44+
45+
- name: upload seed files outside dbt
46+
working-directory: datagen
47+
run: |
48+
python dremio__upload_seeds.py
49+
env:
50+
DREMIO_TOKEN: ${{ secrets.DREMIO_TOKEN }}
51+
DREMIO_FOLDER: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}"
52+
DREMIO_USER: ${{ secrets.DREMIO_USER }}
53+
54+
- name: dbt build
55+
run: dbt build --full-refresh --profiles-dir ./ --profile demo_dremio --exclude "seeds/"
56+
env:
57+
DREMIO_TOKEN: ${{ secrets.DREMIO_TOKEN }}
58+
DREMIO_FOLDER: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}"
59+
DREMIO_USER: ${{ secrets.DREMIO_USER }}
60+
61+
- name: submit artifacts to datafold
62+
run: |
63+
set -ex
64+
datafold dbt upload --ci-config-id 434 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
65+
env:
66+
DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY_BIGQUERY }}
67+
DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}"
68+
GIT_SHA: "${{ github.event.pull_request.head.sha }}"

‎.github/workflows/prod_job.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ jobs:
2121
python-version: '3.8'
2222

2323
- name: install requirements
24-
run: pip install -q -r requirements.txt
24+
run: |
25+
echo "setuptools<72" > constraints.txt
26+
export PIP_CONSTRAINT=constraints.txt
27+
pip install --upgrade pip
28+
pip install -q -r requirements.txt
2529
2630
- name: install datafold-sdk
2731
run: pip install -q datafold-sdk

‎.github/workflows/prod_job_bigquery.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,11 @@ jobs:
4343
python-version: '3.8'
4444

4545
- name: install requirements
46-
run: pip install -q -r requirements.txt
46+
run: |
47+
echo "setuptools<72" > constraints.txt
48+
export PIP_CONSTRAINT=constraints.txt
49+
pip install --upgrade pip
50+
pip install -q -r requirements.txt
4751
4852
- name: install datafold-sdk
4953
run: pip install -q datafold-sdk

‎.github/workflows/prod_job_databricks.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,11 @@ jobs:
4343
python-version: '3.8'
4444

4545
- name: install requirements
46-
run: pip install -q -r requirements.txt
46+
run: |
47+
echo "setuptools<72" > constraints.txt
48+
export PIP_CONSTRAINT=constraints.txt
49+
pip install --upgrade pip
50+
pip install -q -r requirements.txt
4751
4852
- name: install datafold-sdk
4953
run: pip install -q datafold-sdk

0 commit comments

Comments
 (0)