From 2e6f7b8105b0f60c28917c6042daff6e414f6763 Mon Sep 17 00:00:00 2001 From: "dehui.kdh" Date: Thu, 6 Nov 2025 00:15:11 +0800 Subject: [PATCH 1/2] add transparent background models Change-Id: I577d7da1dbcc9bd65c6b5cc60aee6c54f8d32d71 Co-developed-by: Cursor --- src/code/agent/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code/agent/Dockerfile b/src/code/agent/Dockerfile index 34c727e..a36a0b7 100644 --- a/src/code/agent/Dockerfile +++ b/src/code/agent/Dockerfile @@ -35,7 +35,7 @@ ENV PATH="${VENV_DIR}/bin:$PATH" # MNT_DIR is initialized in s.yaml, using env var-'MODEL_ASSET_DIR' -RUN pip install --no-cache-dir mitmproxy huggingface_hub[cli] modelscope +RUN pip install --no-cache-dir huggingface_hub[cli] modelscope COPY . ${AGENT_DIR}/ From c9cd18d2b2f5d3ce22e2339ee4f9927df7fd71d9 Mon Sep 17 00:00:00 2001 From: "dehui.kdh" Date: Thu, 27 Nov 2025 22:11:17 +0800 Subject: [PATCH 2/2] feat: add github proxy Change-Id: Ibf6b13a2ebd7529e3dd26a730f0b6794721c28ac Co-developed-by: Cursor --- src/code/agent/Dockerfile | 6 +- src/code/agent/entrypoint.bash | 98 +++++------ src/code/agent/services/proxy/gitconfig.conf | 35 ++++ .../services/proxy/github-proxy-monitor.sh | 164 ++++++++++++++++++ src/code/agent/services/proxy/mirror_proxy.py | 22 --- src/code/agent/services/proxy/pip.conf | 13 +- src/code/agent/utils/file_ops.py | 104 +++++++++++ 7 files changed, 367 insertions(+), 75 deletions(-) create mode 100644 src/code/agent/services/proxy/gitconfig.conf create mode 100755 src/code/agent/services/proxy/github-proxy-monitor.sh delete mode 100644 src/code/agent/services/proxy/mirror_proxy.py diff --git a/src/code/agent/Dockerfile b/src/code/agent/Dockerfile index a36a0b7..b801cd2 100644 --- a/src/code/agent/Dockerfile +++ b/src/code/agent/Dockerfile @@ -47,7 +47,11 @@ RUN --mount=type=cache,target=/root/.cache/pip \ ${AGENT_DIR}/venv/bin/pip install -r ${AGENT_DIR}/requirements.txt --no-cache-dir RUN mkdir -p ~/.pip \ - && cp ${AGENT_DIR}/services/proxy/pip.conf ~/.pip/pip.conf + && cp ${AGENT_DIR}/services/proxy/pip.conf ~/.pip/pip.conf \ + && cp ${AGENT_DIR}/services/proxy/gitconfig.conf ~/.gitconfig + +# 设置 GitHub 代理监控脚本执行权限 +RUN chmod +x ${AGENT_DIR}/services/proxy/github-proxy-monitor.sh RUN chmod +x ${AGENT_DIR}/entrypoint.bash ENTRYPOINT [ "/bin/bash", "-c", "agent/entrypoint.bash" ] diff --git a/src/code/agent/entrypoint.bash b/src/code/agent/entrypoint.bash index 1378ef0..1e5f200 100644 --- a/src/code/agent/entrypoint.bash +++ b/src/code/agent/entrypoint.bash @@ -19,57 +19,55 @@ # --- 20250102-120159 # ---- comfyui # ---- venv.tar -init_mitmproxy(){ - echo 'export NO_PROXY="127.0.0.1,mirrors.aliyun.com,ghfast.top,ghgo.xyz,ghp.ci,ghproxy.com,hf-mirror.com,deb.debian.org,www.modelscope.cn"'>> ~/.bashrc - echo 'export no_proxy="127.0.0.1,mirrors.aliyun.com,ghfast.top,ghgo.xyz,ghp.ci,ghproxy.com,hf-mirror.com,deb.debian.org,www.modelscope.cn"'>> ~/.bashrc - echo 'export HTTP_PROXY="http://127.0.0.1:8080"' >> ~/.bashrc - echo 'export http_proxy="http://127.0.0.1:8080"' >> ~/.bashrc - - echo 'export HTTPS_PROXY="http://127.0.0.1:8080"' >> ~/.bashrc - echo 'export https_proxy="http://127.0.0.1:8080"' >> ~/.bashrc - - echo 'export HF_ENDPOINT="https://hf-mirror.com"' >> ~/.bashrc - - echo 'export REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"'>> ~/.bashrc - echo 'export SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt"'>> ~/.bashrc - echo 'export CURL_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"'>> ~/.bashrc - - source ~/.bashrc - -# mitmdump -s ${AGENT_DIR}/services/proxy/mirror_proxy.py & - mitmdump -s ${AGENT_DIR}/services/proxy/mirror_proxy.py >> /root/agent/mitmproxy.log 2>> /root/agent/mitmproxy_error.log & - - sleep 5 -# cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/ - cp ~/.mitmproxy/mitmproxy-ca-cert.pem /usr/local/share/ca-certificates/mitmproxy.crt - update-ca-certificates - git config --global http.sslCAInfo /usr/local/share/ca-certificates/mitmproxy.crt -} - -check_and_init_mitmproxy(){ - echo "Check and init mitmproxy..." - - # 根据AUTO_LAUNCH_SNAPSHOT_NAME是否为空来判断当前函数是否use_api_mode - use_api_mode=false - if [[ -n "${AUTO_LAUNCH_SNAPSHOT_NAME}" ]]; then - use_api_mode=true - fi - - # 根据当前REGION判断是否处于国内 - domestic_regions=("cn-hangzhou" "cn-shanghai" "cn-shenzhen" "cn-beijing") - region="${REGION}" - is_domestic=false +# 检查是否为国内区域 +is_domestic_region() { + local domestic_regions=("cn-hangzhou" "cn-shanghai" "cn-shenzhen" "cn-beijing") + local region="${REGION:-}" + for reg in "${domestic_regions[@]}"; do - if [[ "${region}" == "${reg}" ]]; then - is_domestic=true - break - fi + if [[ "${region}" == "${reg}" ]]; then + return 0 + fi done + return 1 +} - if [[ ${use_api_mode} == false ]] && [[ ${is_domestic} == true ]]; then - echo "Init mitmproxy..." - init_mitmproxy +# 设置网络配置 +setup_network() { + echo "[INFO] Setting up network configuration..." + + # 如果是国内集群，配置加速 + if is_domestic_region; then + echo "[INFO] Domestic region detected: ${REGION}" + + # 设置 HuggingFace 镜像 + export HF_ENDPOINT="https://hf-mirror.com" + + # ComfyUI-Manager 使用 + # 设置 Github 镜像 + export GITHUB_ENDPOINT="https://cap-accor-proxy-qkqnjxeail.ap-southeast-1.fcapp.run/https://github.com/" + + # uv 镜像配置 + export UV_DEFAULT_INDEX="https://mirrors.aliyun.com/pypi/simple/" + + # 使用 copy 模式，避免 hardlink 警告（当缓存和目标在不同文件系统时） + export UV_LINK_MODE="copy" + + # 增加超时时间（默认30秒太短） + export UV_HTTP_TIMEOUT="300" + + # 启动 GitHub 代理监控守护进程 + GITHUB_PROXY_MONITOR="${AGENT_DIR}/services/proxy/github-proxy-monitor.sh" + if [ -f "$GITHUB_PROXY_MONITOR" ]; then + echo "[INFO] Starting GitHub proxy monitor..." + nohup bash "$GITHUB_PROXY_MONITOR" > /tmp/github_proxy_monitor.log 2>&1 & + echo "[INFO] GitHub proxy monitor started (logs: /tmp/github_proxy_monitor.log)" + else + echo "[WARN] GitHub proxy monitor not found at: $GITHUB_PROXY_MONITOR" + fi + else + echo "[INFO] Non-domestic region, skipping network acceleration" fi } @@ -92,9 +90,7 @@ mkdir -p ${MNT_DIR}/output source ${AGENT_DIR}/venv/bin/activate echo "Using python venv, python path '$(which python)', pip path '$(which pip)'... " -check_and_init_mitmproxy - -# git diff 忽略文件权限变化，ComfyUI 中 Windows 可执行文件在 Linux 中自动没有可执行权限，导致有 git diff，影响 ComfyUI 版本升级 -git config --global core.fileMode false +# ==================== 网络配置 ==================== +setup_network python ${AGENT_DIR}/main.py \ No newline at end of file diff --git a/src/code/agent/services/proxy/gitconfig.conf b/src/code/agent/services/proxy/gitconfig.conf new file mode 100644 index 0000000..a524ae1 --- /dev/null +++ b/src/code/agent/services/proxy/gitconfig.conf @@ -0,0 +1,35 @@ +# GitHub Proxy Configuration +# 此文件会在容器启动时复制到 ~/.gitconfig + +# 用户信息配置 +[user] + name = FunArt Agent + email = FunArt@comfyui.local + +[url "https://cap-accor-proxy-qkqnjxeail.ap-southeast-1.fcapp.run/https://github.com/"] + insteadOf = https://github.com/ + insteadOf = http://github.com/ + insteadOf = git@github.com: + +# 其他 Git 配置 +[core] + # 忽略文件权限变化 + filemode = false + # 自动处理换行符 + autocrlf = input + +[pull] + # 默认使用 rebase 而不是 merge + rebase = false + +[init] + # 默认分支名 + defaultBranch = main + +# 性能优化 +[gc] + auto = 256 + +[pack] + threads = 0 + diff --git a/src/code/agent/services/proxy/github-proxy-monitor.sh b/src/code/agent/services/proxy/github-proxy-monitor.sh new file mode 100755 index 0000000..6f674f1 --- /dev/null +++ b/src/code/agent/services/proxy/github-proxy-monitor.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# GitHub 代理监控守护进程 +# 功能：定期检测代理可用性，自动切换到可用的代理 +# 工作原理： +# 1. 每 60 秒检测当前代理是否可用 +# 2. 如果不可用，遍历备用代理列表查找可用的 +# 3. 更新 ~/.gitconfig 中的代理配置 + +# ==================== 配置 ==================== + +# 代理列表（按优先级排序） +PROXIES=( + # 测试 github + "https://cap-accor-proxy-qkqnjxeail.ap-southeast-1.fcapp.run/https://github.com/" + "https://cap-accor-proxy-qkqnjxeail.cn-hongkong.fcapp.run/https://github.com/" + "https://gh.llkk.cc/https://github.com/" + "https://ghproxy.com/https://github.com/" + "https://ghfast.top/https://github.com/" +) + +# 日志文件 +LOG_FILE="/tmp/github_proxy_monitor.log" + +# Git 配置文件 +GITCONFIG="$HOME/.gitconfig" + +# 检测间隔（秒） +CHECK_INTERVAL=60 + +# 当前使用的代理 +CURRENT_PROXY="" + +# ==================== 工具函数 ==================== + +log() { + # 只写入日志文件，不输出到 stdout（避免被命令替换捕获） + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE" +} + +# 测试代理是否可用 +test_proxy() { + local proxy_url="$1" + + # 使用 ComfyUI 仓库进行测试（完整的代理 URL） + local test_repo="${proxy_url}comfyanonymous/ComfyUI.git" + + log "🔍 Testing proxy: $proxy_url" + log " Testing repo: $test_repo" + log " Command: GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null git ls-remote --exit-code '$test_repo' HEAD" + + # 使用 git ls-remote 测试（真实的 git 命令） + # GIT_TERMINAL_PROMPT=0: 禁用交互式提示 + # GIT_CONFIG_GLOBAL=/dev/null: 禁用全局配置，避免干扰 + # timeout 10: 10秒超时（git 操作比 curl 慢） + if timeout 10 env GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null git ls-remote --exit-code "$test_repo" HEAD > /dev/null 2>&1; then + log "✅ Proxy test passed: $proxy_url" + return 0 # 成功 + else + log "❌ Proxy test failed: $proxy_url" + return 1 # 失败 + fi +} + +# 查找可用的代理 +find_working_proxy() { + for proxy in "${PROXIES[@]}"; do + if test_proxy "$proxy"; then + log "✅ Found working proxy: $proxy" + echo "$proxy" + return 0 + fi + done + log "⚠️ No working proxy found" + return 1 +} + +# 获取当前 gitconfig 中配置的代理 +get_current_proxy() { + # 读取 gitconfig 中的第一个 url.*.insteadOf 配置 + # 匹配 [url "代理URL"] 格式，提取引号中的 URL + # 注意：这里的 sed 只是解析文本（从管道读取），不会修改 .gitconfig 文件 + local proxy=$(grep -A 1 '^\[url ' "$GITCONFIG" 2>/dev/null | head -1 | sed 's/\[url "\(.*\)"\]/\1/') + + # Debug 日志 + log "🔍 [DEBUG] get_current_proxy() returned: ${proxy:-}" + + echo "$proxy" +} + +# 更新 gitconfig 中的代理配置 +update_gitconfig_proxy() { + local new_proxy="$1" + + log "📝 Updating gitconfig with new proxy: $new_proxy" + + # 备份当前配置 + cp "$GITCONFIG" "${GITCONFIG}.bak" + + # 删除所有现有的 [url ...] 配置段 + # sed 语法：/pattern1/,/pattern2/d 表示删除从 pattern1 到 pattern2 之间的所有行 + # /^\[url / 匹配以 [url 开头的行（配置段开始） + # /^$/ 匹配空行（配置段结束） + # d 删除操作 + # -i.tmp 直接修改文件并创建 .tmp 备份 + sed -i.tmp '/^\[url /,/^$/d' "$GITCONFIG" + + # 添加新的代理配置 + cat >> "$GITCONFIG" << EOF + +# GitHub 代理配置（由 github-proxy-monitor 自动更新） +[url "$new_proxy"] + insteadOf = https://github.com/ + insteadOf = http://github.com/ + insteadOf = git@github.com: +EOF + + # 清理临时文件 + rm -f "${GITCONFIG}.tmp" + + log "✅ Gitconfig updated successfully" +} + +# ==================== 主循环 ==================== + +log "==========================================" +log "GitHub Proxy Monitor started" +log "==========================================" + +# 初始化：读取当前配置的代理 +CURRENT_PROXY=$(get_current_proxy) +if [ -n "$CURRENT_PROXY" ]; then + log "📋 Current proxy from gitconfig: $CURRENT_PROXY" +else + log "⚠️ No proxy configured, will set default proxy" + CURRENT_PROXY="${PROXIES[0]}" + update_gitconfig_proxy "$CURRENT_PROXY" +fi + +# 主监控循环 +while true; do + log "🔍 Checking current proxy: $CURRENT_PROXY" + + if test_proxy "$CURRENT_PROXY"; then + log "✅ Current proxy is working" + else + log "❌ Current proxy failed, searching for alternative..." + + new_proxy=$(find_working_proxy) + log "🔍 [DEBUG] find_working_proxy() returned: ${new_proxy:-}" + + if [ -n "$new_proxy" ]; then + log "⚡ Found working proxy: $new_proxy" + update_gitconfig_proxy "$new_proxy" + CURRENT_PROXY="$new_proxy" + log "🔍 [DEBUG] CURRENT_PROXY updated to: $CURRENT_PROXY" + else + log "⚠️ No working proxy found! Will retry in $CHECK_INTERVAL seconds" + fi + fi + + sleep $CHECK_INTERVAL +done + diff --git a/src/code/agent/services/proxy/mirror_proxy.py b/src/code/agent/services/proxy/mirror_proxy.py deleted file mode 100644 index 5f9f9e4..0000000 --- a/src/code/agent/services/proxy/mirror_proxy.py +++ /dev/null @@ -1,22 +0,0 @@ -import os - -from mitmproxy import http - - -def request(flow: http.HTTPFlow) -> None: - region = os.getenv("REGION") - target_hosts = ["github.com", "huggingface.co", "raw.githubusercontent.com"] - mirror_host = f"{region}.mirrors.functionai.aliyuncs.com" - - if flow.request.pretty_host in target_hosts: - origin = f"{flow.request.scheme}://{flow.request.host}{flow.request.path}" - - # build request to cap mirror - mirror_path = f"/proxy/{flow.request.host}{flow.request.path}" - # print(f"Proxy to cap mirror with mirror_path '{mirror_path}' and origin url '{origin}'", ) - - # proxy request - flow.request.host = mirror_host - flow.request.port = 80 - flow.request.scheme = "http" - flow.request.path = mirror_path diff --git a/src/code/agent/services/proxy/pip.conf b/src/code/agent/services/proxy/pip.conf index 2037824..a5f4b6d 100644 --- a/src/code/agent/services/proxy/pip.conf +++ b/src/code/agent/services/proxy/pip.conf @@ -1,4 +1,15 @@ [global] +# 主镜像源 index-url = https://mirrors.aliyun.com/pypi/simple/ + +# 备用镜像源（按优先级） +extra-index-url = + https://pypi.tuna.tsinghua.edu.cn/simple/ + https://pypi.mirrors.ustc.edu.cn/simple/ + [install] -trusted-host = mirrors.aliyun.com \ No newline at end of file +# 信任的主机列表 +trusted-host = + mirrors.aliyun.com + pypi.tuna.tsinghua.edu.cn + pypi.mirrors.ustc.edu.cn \ No newline at end of file diff --git a/src/code/agent/utils/file_ops.py b/src/code/agent/utils/file_ops.py index de0af9d..381a03c 100644 --- a/src/code/agent/utils/file_ops.py +++ b/src/code/agent/utils/file_ops.py @@ -195,6 +195,110 @@ def create_symlink(source_path, link_path, force=False): print(f"Failed to create symlink from {source_path} to {link_path}, reason: {e}") raise +def compress_with_zstd(target_file_path, source_dir, selected_files=None, level=10): + """ + 使用 zstd 打包文件/文件夹为 tar.zst 格式 + + Args: + target_file_path: 输出的 .tar.zst 文件路径 + source_dir: 待打包文件所在父目录 + selected_files: 父目录下，待打包文件/文件夹名列表，None表示全部打包 + level: zstd 压缩级别（1-22，默认 10，平衡压缩比和速度） + """ + import subprocess + + # 记录开始压缩 + files_desc = ', '.join(selected_files) if selected_files else 'all files' + print(f"Using zstd compression (level {level}) for {files_desc} -> {target_file_path}") + + try: + # 保存当前工作目录 + original_dir = os.getcwd() + try: + os.chdir(source_dir) + + # 构建文件列表 + files_str = ' '.join(selected_files) if selected_files else '.' + + # 使用 shell 命令直接执行（简洁且高效） + # tar 参数说明: + # -c: 创建新归档文件 + # -f -: 输出到标准输出（stdout），通过管道传给 zstd + # zstd 参数说明: + # -{level}: 压缩级别（1-22，默认 10） + # -T0: 使用多线程（0 表示使用所有 CPU 核心） + # -o: 指定输出文件路径 + cmd = f"tar -cf - {files_str} | zstd -{level} -T0 -o {target_file_path}" + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + check=False + ) + + if result.returncode != 0: + raise Exception(f"Compression failed: {result.stderr}") + + print(f"Successfully compressed with zstd: {target_file_path}") + + finally: + # 恢复原始工作目录 + os.chdir(original_dir) + + except Exception as e: + print(f"Failed to compress with zstd to {target_file_path}, reason: {e}") + raise + + +def extract_zstd(file_path, output_dir=None): + """ + 解压 tar.zst 包 + + Args: + file_path: .tar.zst 压缩文件路径 + output_dir: 解压输出目录，默认为None，表示解压到压缩文件所在目录 + """ + import subprocess + + # 如果未指定输出目录，使用压缩文件所在目录 + if output_dir is None: + output_dir = os.path.dirname(file_path) + + # 确保输出目录存在 + os.makedirs(output_dir, exist_ok=True) + + # 记录开始解压 + print(f"Using zstd decompression for {file_path} -> {output_dir}") + + try: + # 使用 shell 命令直接执行（简洁且高效） + # zstd 参数说明: + # -d: 解压模式 + # -c: 输出到标准输出（stdout），通过管道传给 tar + # tar 参数说明: + # -x: 解压归档文件 + # -f -: 从标准输入（stdin）读取数据 + # -C: 指定解压目标目录 + cmd = f"zstd -d -c {file_path} | tar -xf - -C {output_dir}" + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + check=False + ) + + if result.returncode != 0: + raise Exception(f"Extraction failed: {result.stderr}") + + print(f"Successfully decompressed with zstd: {file_path}") + + except Exception as e: + print(f"Failed to extract zstd archive {file_path}, reason: {e}") + raise + + def robust_readlines(fullpath): """ 健壮的文件读取方法，可处理不同编码的文本文件，当前业务场景下主要用于读取requirements.txt