Skip to content

Commit 3ef4326

Browse files
authored
Merge branch 'main' into fix/backfill-race-61375
2 parents 3372139 + 2525023 commit 3ef4326

File tree

290 files changed

+12470
-5165
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

290 files changed

+12470
-5165
lines changed

.devcontainer/devcontainer.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
"../scripts/ci/docker-compose/devcontainer.yml",
55
"../scripts/ci/docker-compose/devcontainer-sqlite.yml"
66
],
7+
"features": {
8+
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
9+
},
710
"settings": {
811
"terminal.integrated.defaultProfile.linux": "bash"
912
},

.devcontainer/mysql/devcontainer.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
"../../scripts/ci/docker-compose/backend-mysql.yml",
66
"../../scripts/ci/docker-compose/devcontainer-mysql.yml"
77
],
8+
"features": {
9+
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
10+
},
811
"settings": {
912
"terminal.integrated.defaultProfile.linux": "bash"
1013
},

.devcontainer/postgres/devcontainer.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
"../../scripts/ci/docker-compose/backend-postgres.yml",
66
"../../scripts/ci/docker-compose/devcontainer-postgres.yml"
77
],
8+
"features": {
9+
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
10+
},
811
"settings": {
912
"terminal.integrated.defaultProfile.linux": "bash"
1013
},

.github/CODEOWNERS

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,5 +150,12 @@ Dockerfile.ci @potiuk @ashb @gopidesupavan @amoghrajesh @jscheffl @bugraoz93 @ka
150150
# Shared Libraries
151151
/shared/ @ashb @amoghrajesh @potiuk
152152

153+
# Agentic instructions
154+
/AGENTS.md @potiuk @kaxil @jscheffl @amoghrajesh @ashb @sjyangkevin
155+
/.github/instructions/ @potiuk @kaxil @jscheffl @amoghrajesh @ashb @sjyangkevin
156+
/airflow-core/src/airflow/api_fastapi/execution_api/AGENTS.md @potiuk @kaxil @jscheffl @amoghrajesh @ashb @sjyangkevin
157+
/providers/AGENTS.md @potiuk @kaxil @jscheffl @amoghrajesh @ashb @sjyangkevin
158+
/.github/skills/ @potiuk @kaxil @jscheffl @amoghrajesh @ashb @sjyangkevin
159+
153160
# RMs on release documents
154161
/dev/README_RELEASE_*.md @potiuk @jscheffl @vincbeck @shahar1 @jedcunningham @bugraoz93

.github/actions/breeze/action.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ inputs:
2424
default: "3.10"
2525
uv-version:
2626
description: 'uv version to use'
27-
default: "0.10.6" # Keep this comment to allow automatic replacement of uv version
27+
default: "0.10.7" # Keep this comment to allow automatic replacement of uv version
2828
outputs:
2929
host-python-version:
3030
description: Python version used in host
@@ -38,7 +38,7 @@ runs:
3838
python-version: ${{ inputs.python-version }}
3939
- name: "Install uv"
4040
shell: bash
41-
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
41+
run: pip install "uv==${UV_VERSION}"
4242
env:
4343
UV_VERSION: ${{ inputs.uv-version }}
4444
# NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has

.github/actions/install-prek/action.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ inputs:
2424
default: "3.10"
2525
uv-version:
2626
description: 'uv version to use'
27-
default: "0.10.6" # Keep this comment to allow automatic replacement of uv version
27+
default: "0.10.7" # Keep this comment to allow automatic replacement of uv version
2828
prek-version:
2929
description: 'prek version to use'
30-
default: "0.3.3" # Keep this comment to allow automatic replacement of prek version
30+
default: "0.3.4" # Keep this comment to allow automatic replacement of prek version
3131
save-cache:
3232
description: "Whether to save prek cache"
3333
required: true
@@ -39,7 +39,7 @@ runs:
3939
steps:
4040
- name: "Install uv"
4141
shell: bash
42-
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
42+
run: pip install "uv==${UV_VERSION}"
4343
env:
4444
UV_VERSION: ${{ inputs.uv-version }}
4545
- name: Install prek

.github/instructions/code-review.instructions.md

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -10,45 +10,49 @@ Use these rules when reviewing pull requests to the Apache Airflow repository.
1010
## Architecture Boundaries
1111

1212
- **Scheduler must never run user code.** It only processes serialized Dags. Flag any scheduler-path code that deserializes or executes Dag/task code.
13-
- **Workers must not access the metadata DB directly.** Task execution communicates with the API server through the Execution API (`/execution` endpoints) only.
14-
- **Dag Processor and Triggerer run user code in isolated processes.** Code in these components should maintain that isolation.
15-
- **Providers must not import core internals** like `SUPERVISOR_COMMS` or task-runner plumbing. Providers interact through the public SDK and execution API only.
13+
- **Flag any task execution code that accesses the metadata DB directly** instead of through the Execution API (`/execution` endpoints).
14+
- **Flag any code in Dag Processor or Triggerer that breaks process isolation** these components run user code in isolated processes.
15+
- **Flag any provider importing core internals** like `SUPERVISOR_COMMS` or task-runner plumbing. Providers interact through the public SDK and execution API only.
1616

1717
## Database and Query Correctness
1818

19-
- **N+1 queries**: Flag SQLAlchemy queries that access relationships inside loops without `joinedload()` or `selectinload()`.
20-
- **`run_id` is only unique per Dag.** Queries that group, partition, or join on `run_id` alone (without `dag_id`) will collide across Dags. Always require `(dag_id, run_id)` together.
21-
- **Cross-database compatibility**: SQL changes must work on PostgreSQL, MySQL, and SQLite. Flag database-specific features (lateral joins, window functions) without cross-DB handling.
22-
- **Session discipline**: In `airflow-core`, functions receiving a `session` parameter must not call `session.commit()`. Use keyword-only `session` parameters.
19+
- **Flag any SQLAlchemy relationship access inside a loop** without `joinedload()` or `selectinload()` — this is an N+1 query.
20+
- **Flag any query on `run_id` without `dag_id`.** `run_id` is only unique per Dag. Queries that filter, group, partition, or join on `run_id` alone will silently collide across Dags.
21+
- **Flag any `session.commit()` call in `airflow-core`** code that receives a `session` parameter. Session lifecycle is managed by the caller, not the callee.
22+
- **Flag any `session` parameter that is not keyword-only** (`*, session`) in `airflow-core`.
23+
- **Flag any database-specific SQL** (e.g., `LATERAL` joins, PostgreSQL-only functions, MySQL-only syntax) without cross-DB handling. SQL must work on PostgreSQL, MySQL, and SQLite.
2324

2425
## Code Quality Rules
2526

26-
- No `assert` in production code (stripped in optimized Python).
27-
- `time.monotonic()` for durations, not `time.time()`.
28-
- Imports at top of file. Valid exceptions: circular imports, lazy loading for worker isolation, `TYPE_CHECKING` blocks.
29-
- Guard heavy type-only imports (e.g., `kubernetes.client`) with `TYPE_CHECKING` in multi-process code paths.
30-
- Unbounded caches are bugs: all `@lru_cache` must have `maxsize`.
31-
- Resources (files, connections, sessions) must use context managers or `try/finally`.
27+
- **Flag any `assert` in non-test code.** `assert` is stripped in optimized Python (`python -O`), making it a silent no-op in production.
28+
- **Flag any `time.time()` used for measuring durations.** Use `time.monotonic()` instead — `time.time()` is affected by system clock adjustments.
29+
- **Flag any `from` or `import` statement inside a function or method body.** Imports must be at the top of the file. The only valid exceptions are: (1) circular import avoidance, (2) lazy loading for worker isolation, (3) `TYPE_CHECKING` blocks. If the import is inside a function, ask the author to justify why it cannot be at module level.
30+
- **Flag any `@lru_cache(maxsize=None)`.** This creates an unbounded cache — every unique argument set is cached forever. Note: `@lru_cache()` without arguments defaults to `maxsize=128` and is fine.
31+
- **Flag any heavy import** (e.g., `kubernetes.client`) in multi-process code paths that is not behind a `TYPE_CHECKING` guard.
32+
- **Flag any file, connection, or session opened without a context manager or `try/finally`.**
3233

3334
## Testing Requirements
3435

35-
- New behavior requires tests covering success, failure, and edge cases.
36-
- Use pytest patterns, not `unittest.TestCase`.
37-
- Use `spec`/`autospec` when mocking.
38-
- Use `time_machine` for time-dependent tests.
39-
- Imports belong at the top of test files, not inside test functions.
40-
- Issue numbers do not belong in test docstrings.
36+
- **Flag any new public method or behavior without corresponding tests.** Tests must cover success, failure, and edge cases.
37+
- **Flag any `unittest.TestCase` subclass.** Use pytest patterns instead.
38+
- **Flag any `mock.Mock()` or `mock.MagicMock()` without `spec` or `autospec`.** Unspec'd mocks silently accept any attribute access, hiding real bugs.
39+
- **Flag any `time.sleep` or `datetime.now()` in tests.** Use `time_machine` for time-dependent tests.
40+
- **Flag any issue number in test docstrings** (e.g., `"""Fix for #12345"""`) — test names should describe behavior, not track tickets.
4141

4242
## API Correctness
4343

44-
- `map_index` must be handled correctly for mapped tasks. Queries without `map_index` filtering may return arbitrary task instances.
45-
- Execution API changes must follow Cadwyn versioning (CalVer format).
44+
- **Flag any query on mapped task instances that does not filter on `map_index`.** Without it, queries may return arbitrary instances from the mapped set.
45+
- **Flag any Execution API change without a Cadwyn version migration** (CalVer format).
4646

4747
## UI Code (React/TypeScript)
4848

4949
- Avoid `useState + useEffect` to sync derived state. Use nullish coalescing or nullable override patterns instead.
5050
- Extract shared logic into custom hooks rather than copy-pasting across components.
5151

52+
## Generated Files
53+
54+
- **Flag any manual edits to files in `openapi-gen/`** or Task SDK generated models. These must be regenerated, not hand-edited.
55+
5256
## AI-Generated Code Signals
5357

5458
Flag these patterns that indicate low-quality AI-generated contributions:
@@ -63,7 +67,5 @@ Flag these patterns that indicate low-quality AI-generated contributions:
6367

6468
## Quality Signals to Check
6569

66-
The absence of these signals in a "fix" or "optimization" PR is itself a red flag:
67-
68-
- **Bug fixes need regression tests**: A test that fails without the fix and passes with it.
69-
- **Existing tests must still pass without modification**: If existing tests need changes to pass, the PR may introduce a behavioral regression.
70+
- **For bug-fix PRs, flag if there is no regression test** — a test that fails without the fix and passes with it.
71+
- **Flag any existing test modified to accommodate new behavior** — this may indicate a behavioral regression rather than a genuine fix.

.github/workflows/airflow-distributions-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ jobs:
115115
# Pin on virtualenv is temporary for pypa/hatch#2193
116116
run: |
117117
uv tool uninstall hatch || true
118-
uv tool install hatch==1.16.4 --with 'virtualenv<21'
118+
uv tool install hatch==1.16.5
119119
breeze release-management "${DISTRIBUTION_TYPE}" --distribution-format wheel
120120
if: ${{ matrix.python-version == inputs.default-python-version }}
121121
- name: "Verify wheel packages with twine"

.github/workflows/basic-tests.yml

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ on: # yamllint disable-line rule:truthy
7070
type: string
7171
uv-version:
7272
description: 'uv version to use'
73-
default: "0.10.6" # Keep this comment to allow automatic replacement of uv version
73+
default: "0.10.7" # Keep this comment to allow automatic replacement of uv version
7474
type: string
7575
platform:
7676
description: 'Platform for the build - linux/amd64 or linux/arm64'
@@ -139,7 +139,7 @@ jobs:
139139
fetch-depth: 1
140140
persist-credentials: false
141141
- name: "Install uv"
142-
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
142+
run: pip install "uv==${UV_VERSION}"
143143
env:
144144
UV_VERSION: ${{ inputs.uv-version }}
145145
- name: "Run shared ${{ matrix.shared-distribution }} tests"
@@ -329,16 +329,18 @@ jobs:
329329
--hook-stage manual upgrade-important-versions || true
330330
if: always()
331331
env:
332-
UPGRADE_PIP: "false"
333-
UPGRADE_PYTHON: "false"
332+
UPGRADE_FLIT: "false"
333+
UPGRADE_GITPYTHON: "false"
334334
UPGRADE_GOLANG: "false"
335-
UPGRADE_NODE_LTS: "false"
336335
UPGRADE_HATCH: "false"
336+
UPGRADE_HATCHLING: "false"
337+
UPGRADE_MYPY: "false"
338+
UPGRADE_NODE_LTS: "false"
339+
UPGRADE_PIP: "false"
340+
UPGRADE_PYTHON: "false"
337341
UPGRADE_PYYAML: "false"
338-
UPGRADE_GITPYTHON: "false"
339342
UPGRADE_RICH: "false"
340343
UPGRADE_RUFF: "false"
341-
UPGRADE_MYPY: "false"
342344
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
343345
- name: "Run automated upgrade for important versions minus uv (failing if needed)"
344346
run: |
@@ -425,7 +427,7 @@ jobs:
425427
with:
426428
persist-credentials: false
427429
- name: "Install uv"
428-
run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
430+
run: pip install "uv==${UV_VERSION}"
429431
env:
430432
UV_VERSION: ${{ inputs.uv-version }}
431433
- name: "Set up Airflow home directory"

.github/workflows/ci-amd-arm.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ env:
4040
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4141
GITHUB_USERNAME: ${{ github.actor }}
4242
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
43+
UV_VERSION: "0.10.7" # Keep this comment to allow automatic replacement of uv version
4344
VERBOSE: "true"
4445

4546
concurrency:
@@ -248,7 +249,7 @@ jobs:
248249
with:
249250
persist-credentials: false
250251
- name: "Install uv"
251-
run: curl -LsSf https://astral.sh/uv/install.sh | sh
252+
run: pip install "uv==${UV_VERSION}"
252253
- name: "Verify release calendar"
253254
run: uv run dev/verify_release_calendar.py
254255

0 commit comments

Comments
 (0)