Merge tag 'v0.21.1' into upgrade-0.21.1

Author: sd109

.github/ISSUE_TEMPLATE/agent_scenario_request.yml

@@ -0,0 +1,46 @@
+name: "❤️‍🔥ᴬᴳᴱᴺᵀ Agent scenario request"
+description: Propose a agent scenario request for RAGFlow.
+title: "[Agent Scenario Request]: "
+labels: ["❤️‍🔥ᴬᴳᴱᴺᵀ agent scenario"]
+body:
+  - type: checkboxes
+    attributes:
+      label: Self Checks
+      description: "Please check the following in order to be responded in time :)"
+      options:
+        - label: I have searched for existing issues [search for existing issues](https://github.com/infiniflow/ragflow/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit this report ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: Non-english title submitions will be closed directly ( 非英文标题的提交将会被直接关闭 ) ([Language Policy](https://github.com/infiniflow/ragflow/issues/5910)).
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
+          required: true
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a scenario?
+      description: |
+        A clear and concise description of what the scenario is. Ex. I'm always frustrated when [...]
+      render: Markdown
+    validations:
+      required: false
+  - type: textarea
+    attributes:
+      label: Describe the feature you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Documentation, adoption, use case
+      description: If you can, explain some scenarios how users might use this, situations it would be helpful in. Any API designs, mockups, or diagrams are also helpful.
+      render: Markdown
+    validations:
+      required: false
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: |
+        Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
           fetch-depth: 0
           fetch-tags: true
 
@@ -69,7 +69,7 @@ jobs:
         # https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
         uses: softprops/action-gh-release@v2
         with:
-          token: ${{ secrets.MY_GITHUB_TOKEN }}  # Use the secret as an environment variable
+          token: ${{ secrets.GITHUB_TOKEN }}  # Use the secret as an environment variable
           prerelease: ${{ env.PRERELEASE }}
           tag_name: ${{ env.RELEASE_TAG }}
           # The body field does not support environment variable substitution directly.
@@ -88,7 +88,9 @@ jobs:
         with:
           context: .
           push: true
-          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
+          tags: |
+            infiniflow/ragflow:${{ env.RELEASE_TAG }}
+            infiniflow/ragflow:latest-full
           file: Dockerfile
           platforms: linux/amd64
 
@@ -98,7 +100,9 @@ jobs:
         with:
           context: .
           push: true
-          tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
+          tags: |
+            infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
+            infiniflow/ragflow:latest-slim
           file: Dockerfile
           build-args: LIGHTEN=1
           platforms: linux/amd64
@@ -116,3 +120,17 @@ jobs:
           packages-dir: sdk/python/dist/
           password: ${{ secrets.PYPI_API_TOKEN }}
           verbose: true
+
+      - name: Build ragflow-cli
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          cd admin/client && \
+          uv build
+
+      - name: Publish client package distributions to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: admin/client/dist/
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true

.github/workflows/tests.yml

@@ -34,12 +34,10 @@ jobs:
       # https://github.com/hmarr/debug-action
       #- uses: hmarr/debug-action@v2
 
-      - name: Show who triggered this workflow
+      - name: Ensure workspace ownership
         run: |
           echo "Workflow triggered by ${{ github.event_name }}"
-
-      - name: Ensure workspace ownership
-        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+          echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
 
       # https://github.com/actions/checkout/issues/1781
       - name: Check out code
@@ -48,6 +46,44 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      - name: Check workflow duplication
+        if: ${{ !cancelled() && !failure() && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci')) }}
+        run: |
+          if [[ ${{ github.event_name }} != 'pull_request' ]]; then
+            HEAD=$(git rev-parse HEAD)
+            # Find a PR that introduced a given commit
+            gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
+            PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
+            echo "HEAD=${HEAD}"
+            echo "PR_NUMBER=${PR_NUMBER}"
+            if [[ -n ${PR_NUMBER} ]]; then
+              PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
+              if [[ -f ${PR_SHA_FP} ]]; then
+                read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
+                # Calculate the hash of the current workspace content
+                HEAD_SHA=$(git rev-parse HEAD^{tree})
+                if [[ ${HEAD_SHA} == ${PR_SHA} ]]; then
+                  echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
+                  gh run cancel ${GITHUB_RUN_ID}
+                  while true; do
+                    status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status)
+                    [ "$status" = "completed" ] && break
+                    sleep 5
+                  done
+                  exit 1
+                fi
+              fi
+            fi
+          else
+            PR_NUMBER=${{ github.event.pull_request.number }}
+            PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
+            # Calculate the hash of the current workspace content
+            PR_SHA=$(git rev-parse HEAD^{tree})
+            echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}"
+            mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}
+            echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP}
+          fi
+
       # https://github.com/astral-sh/ruff-action
       - name: Static check with Ruff
         uses: astral-sh/ruff-action@v3
@@ -59,14 +95,15 @@ jobs:
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
           sudo docker pull ubuntu:22.04
-          sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
 
       - name: Build ragflow:nightly
         run: |
-          sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
 
       - name: Start ragflow:nightly-slim
         run: |
+          sudo docker compose -f docker/docker-compose.yml down --volumes --remove-orphans
           echo -e "\nRAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
           sudo docker compose -f docker/docker-compose.yml up -d
 

.gitignore

@@ -149,7 +149,7 @@ out
 # Nuxt.js build / generate output
 .nuxt
 dist
-
+ragflow_cli.egg-info
 # Gatsby files
 .cache/
 # Comment in the public line in if your project uses Gatsby and not Next.js
@@ -193,3 +193,5 @@ dist
 # SvelteKit build / generate output
 .svelte-kit
 
+# Default backup dir
+backup

.trivyignore

@@ -0,0 +1,15 @@
+**/*.md
+**/*.min.js
+**/*.min.css
+**/*.svg
+**/*.png
+**/*.jpg
+**/*.jpeg
+**/*.gif
+**/*.woff
+**/*.woff2
+**/*.map
+**/*.webp
+**/*.ico
+**/*.ttf
+**/*.eot

Dockerfile

@@ -191,6 +191,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 ENV PYTHONPATH=/ragflow/
 
 COPY web web
+COPY admin admin
 COPY api api
 COPY conf conf
 COPY deepdoc deepdoc

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 <a href="https://demo.ragflow.io/">
-<img src="web/src/assets/logo-with-text.png" width="520" alt="ragflow logo">
+<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
 </a>
 </div>
 
@@ -22,7 +22,7 @@
         <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.19.1">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.21.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -71,22 +71,22 @@
 
 ## 💡 What is RAGFlow?
 
-[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document
-understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models)
-to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted
-data.
+[RAGFlow](https://ragflow.io/) is a leading open-source Retrieval-Augmented Generation (RAG) engine that fuses cutting-edge RAG with Agent capabilities to create a superior context layer for LLMs. It offers a streamlined RAG workflow adaptable to enterprises of any scale. Powered by a converged context engine and pre-built agent templates, RAGFlow enables developers to transform complex data into high-fidelity, production-ready AI systems with exceptional efficiency and precision.
 
 ## 🎮 Demo
 
 Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/7248/2f6baa3e-1092-4f11-866d-36f6a9d075e5" width="1200"/>
-<img src="https://github.com/user-attachments/assets/504bbbf1-c9f7-4d83-8cc5-e9cb63c26db6" width="1200"/>
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
+<img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/agentic-dark.gif" width="1200"/>
 </div>
 
 ## 🔥 Latest Updates
 
+- 2025-10-15 Supports orchestrable ingestion pipeline.
+- 2025-08-08 Supports OpenAI's latest GPT-5 series models.
+- 2025-08-01 Supports agentic workflow and MCP.
 - 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
 - 2025-05-05 Supports cross-language query.
 - 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
@@ -135,7 +135,7 @@ releases! 🌟
 ## 🔎 System Architecture
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
-<img src="https://github.com/infiniflow/ragflow/assets/12318111/d6ac5664-c237-4200-a7c2-a4a00691b485" width="1000"/>
+<img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
 ## 🎬 Get Started
@@ -187,7 +187,7 @@ releases! 🌟
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-   > The command below downloads the `v0.19.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.19.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.19.1` for the full edition `v0.19.1`.
+   > The command below downloads the `v0.21.1-slim` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.21.1-slim`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. For example: set `RAGFLOW_IMAGE=infiniflow/ragflow:v0.21.1` for the full edition `v0.21.1`.
 
    ```bash
    $ cd ragflow/docker
@@ -200,8 +200,8 @@ releases! 🌟
 
    | RAGFlow image tag | Image size (GB) | Has embedding models? | Stable?                  |
    |-------------------|-----------------|-----------------------|--------------------------|
-   | v0.19.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
-   | v0.19.1-slim      | &approx;2       | ❌                   | Stable release            |
+   | v0.21.1           | &approx;9       | :heavy_check_mark:    | Stable release           |
+   | v0.21.1-slim      | &approx;2       | ❌                   | Stable release            |
    | nightly           | &approx;9       | :heavy_check_mark:    | _Unstable_ nightly build |
    | nightly-slim      | &approx;2       | ❌                   | _Unstable_ nightly build  |
 
@@ -304,7 +304,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
 
 ## 🔨 Launch service from source for development
 
-1. Install uv, or skip this step if it is already installed:
+1. Install `uv` and `pre-commit`, or skip this step if they are already installed:
 
    ```bash
    pipx install uv pre-commit
@@ -341,12 +341,16 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
 5. If your operating system does not have jemalloc, please install it as follows:
 
    ```bash
-   # ubuntu
+   # Ubuntu
    sudo apt-get install libjemalloc-dev
-   # centos
+   # CentOS
    sudo yum install jemalloc
+   # OpenSUSE
+   sudo zypper install jemalloc
+   # macOS
+   sudo brew install jemalloc
    ```
-   
+
 6. Launch backend service:
 
    ```bash