Merge branch 'main' into amrit/storage-cli

Author: amritghimire

.github/workflows/benchmarks.yml

@@ -18,7 +18,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
         with:

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 

.github/workflows/tests-studio.yml

@@ -62,15 +62,15 @@ jobs:
           echo "Studio branch: $STUDIO_BRANCH"
 
       - name: Check out Studio
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           repository: iterative/studio
           ref: ${{ env.STUDIO_BRANCH }}
           token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
 
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: './backend/datachain'
           fetch-depth: 0

.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
@@ -73,11 +73,29 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
+      - name: Setup PostgreSQL
+        if: runner.os != 'Windows'
+        uses: ikalnytskyi/action-setup-postgres@10ab8a56cc77b4823c2bfa57b1d4dd5605ef0481 # v7
+        with:
+          username: test
+          password: test
+          database: test_datachain
+          port: 5432
+          postgres-version: "17"
+        id: postgres
+
+      - name: Set PostgreSQL URI
+        if: runner.os != 'Windows'
+        run: |
+          FULL_URI="${{ steps.postgres.outputs.connection-uri }}"
+          echo "TEST_POSTGRES_URI=${FULL_URI%/*}" >> "$GITHUB_ENV"
+        shell: bash
+
       - name: Set up Python ${{ matrix.pyv }}
         uses: actions/setup-python@v5
         with:
@@ -157,7 +175,7 @@ jobs:
           - {os: ubuntu-latest-4-cores, pyv: "3.13", group: multimodal}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
@@ -176,6 +194,22 @@ jobs:
       - name: Install nox
         run: uv pip install nox --system
 
+      - name: Install FFmpeg on Windows
+        if: runner.os == 'Windows'
+        run: choco install ffmpeg
+
+      - name: Install FFmpeg on macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install ffmpeg
+          echo 'DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib' >> "$GITHUB_ENV"
+
+      - name: Install FFmpeg on Ubuntu
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
+
       - name: Set hf token
         if: matrix.group == 'llm_and_nlp'
         run: echo 'HF_TOKEN=${{ secrets.HF_TOKEN }}' >> "$GITHUB_ENV"

.github/workflows/update-template.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Update template
         uses: iterative/py-template@main

.gitignore

@@ -145,3 +145,7 @@ cython_debug/
 *.pt
 
 .DS_Store/
+
+# for local dev, e.g. LLM generated files, .env.test to override
+# test variables, local scripts to try, etc
+local/

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.8'
+    rev: 'v0.12.10'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

docs/references/func.md

@@ -1,5 +1,38 @@
 # Functions
 
-Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate). Import these functions from `datachain.func`.
+Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate).
 
-::: datachain.func
+Functions are organized by category and accessed through their respective modules. For example, string functions are accessed via `func.string.length()`, array functions via `func.array.contains()`, etc.
+
+!!! note "Global Function Access"
+    Only a subset of functions are available directly from `datachain.func` (e.g., `func.length`). Most functions should be accessed through their specific module namespace (e.g., `func.string.length`) to avoid naming conflicts.
+
+## Function Categories
+
+DataChain provides several categories of functions for different types of operations:
+
+- **[Aggregate Functions](functions/aggregate.md)** - Functions for aggregating data like `sum`, `count`, `avg`, etc.
+- **[Array Functions](functions/array.md)** - Functions for working with arrays and lists
+- **[Conditional Functions](functions/conditional.md)** - Functions for conditional logic like `ifelse`, `case`, etc.
+- **[Numeric Functions](functions/numeric.md)** - Functions for numeric operations and computations
+- **[Path Functions](functions/path.md)** - Functions for working with file paths
+- **[Random Functions](functions/random.md)** - Functions for generating random values
+- **[String Functions](functions/string.md)** - Functions for string manipulation and processing
+- **[Window Functions](functions/window.md)** - Functions for window operations
+
+## Usage
+
+```python
+from datachain.func import aggregate, array, conditional, numeric, path, random, string, window
+
+# Access functions through their module namespaces
+dc.mutate(
+    text_length=string.length("text_column"),
+    contains_item=array.contains("array_column", "value"),
+    file_extension=path.file_ext("file_path")
+)
+
+# Some commonly used functions are also available directly
+from datachain.func import sum, count, length, ifelse
+dc.mutate(total=sum("amount"))
+```

docs/references/functions/aggregate.md

@@ -0,0 +1,5 @@
+# Aggregate Functions
+
+Aggregate functions perform calculations on sets of values and return a single result.
+
+::: datachain.func.aggregate

docs/references/functions/array.md

@@ -0,0 +1,5 @@
+# Array Functions
+
+Functions for working with arrays and lists, including operations like distance calculations and array manipulation.
+
+::: datachain.func.array