diff --git a/.augment-guidelines b/.augment-guidelines new file mode 100644 index 0000000000000..79e2b5b32f305 --- /dev/null +++ b/.augment-guidelines @@ -0,0 +1,62 @@ +# Augment Guidelines for FFmpeg Repository + +project: + name: "FFmpeg" + description: | + A complete, cross-platform solution to record, convert and stream audio and video. + type: "media" + +architecture: + overview: | + FFmpeg is organized into several core libraries that handle different aspects of + multimedia processing. The codebase follows a modular design where each library + can be used independently or together. FFmpeg also provides command-line tools + built on top of these libraries. + + key_directories: + - path: "libavcodec" + description: | + Implements encoders and decoders for audio/video codecs and bitstream processing + - path: "libavdevice" + description: | + Provides abstraction for accessing capture and playback devices + - path: "libavfilter" + description: | + Implements media filtering framework for processing audio and video + - path: "libavformat" + description: | + Handles container formats, muxing/demuxing, and I/O protocols + - path: "libavutil" + description: | + Contains utility functions, data structures, and common components shared across + libraries + - path: "libswresample" + description: | + Implements audio mixing and resampling routines + - path: "tests" + description: | + Contains test suites and validation tools for FFmpeg functionality + +components: + - name: "WHIP" + description: | + WebRTC-HTTP Ingestion Protocol implementation for low-latency streaming. Handles + SDP exchange, ICE connectivity, DTLS handshake, SRTP encryption, and RTP + packetization for WebRTC streaming. + related_files: + - path: "libavformat/whip.c" + description: | + Core implementation of the WHIP protocol, including SDP offer/answer exchange, + ICE connectivity, DTLS handshake setup, and SRTP encryption for RTP packets + - path: "libavformat/tls.h" + description: | + Header defining the DTLS interface used by WHIP for secure communication, + including functions for certificate handling and DTLS state management + - path: "libavformat/tls.c" + description: | + Common DTLS implementation shared across different SSL backends, providing + UDP socket setup for DTLS connections + - path: "libavformat/tls_openssl.c" + description: | + OpenSSL-specific implementation of DTLS functionality, including handshake + procedures and SRTP key material export diff --git a/.augmentignore b/.augmentignore new file mode 100644 index 0000000000000..0ec86037100e4 --- /dev/null +++ b/.augmentignore @@ -0,0 +1,25 @@ + +# Build artifacts +**/objs/** +**/build/** +**/*.o +**/*.a +**/*.so +**/*.dylib +**/*.d + +# IDE files +**/.idea/** +**/.vscode/** +**/.run/** + +# Generated files +**/.tmp/** +**/fate-suite/** +**/*.flv +**/*.mp4 +**/*.ts + +# Other files. +**/tools/** +**/tests/** diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile new file mode 100644 index 0000000000000..78e8d928afe2f --- /dev/null +++ b/.github/docker/Dockerfile @@ -0,0 +1,24 @@ +# docker build -t ossrs/srs:ffmpeg-fate +# docker push ossrs/srs:ffmpeg-fate +FROM ubuntu:22.04 + +RUN apt-get update && \ + apt-get install -y build-essential git rsync make nasm pkg-config libssl-dev &&\ + rm -rf /var/lib/apt/lists/* + +WORKDIR /opt +RUN git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg + +WORKDIR /opt/ffmpeg +RUN ./configure --enable-openssl --enable-version3 +RUN make -j$(nproc) + +RUN make fate-rsync SAMPLES=/opt/ffmpeg/fate-suite +RUN du -sh /opt/ffmpeg/fate-suite + +# Note that you should use the fate-suite.tar, then extract it out of +# docker, to avoid resync all files. +RUN tar cf fate-suite.tar fate-suite +RUN du -sh /opt/ffmpeg/fate-suite.tar + +ENV FATE_SAMPLES=/opt/ffmpeg/fate-suite diff --git a/.github/scripts/format-patch.sh b/.github/scripts/format-patch.sh new file mode 100755 index 0000000000000..3b56e701a652e --- /dev/null +++ b/.github/scripts/format-patch.sh @@ -0,0 +1,189 @@ +#!/bin/bash + +LOGPREFIX=">>" + +if [[ $(jq --version 1>/dev/null 2>&1 && echo yes) != "yes" ]]; then + echo "${LOGPREFIX} Tool jq is not installed. Please install it to parse JSON data. For example:" + echo "${LOGPREFIX} apt install jq" + echo "${LOGPREFIX} brew install jq" + echo "${LOGPREFIX} yum install jq" + echo "${LOGPREFIX} See https://github.com/jqlang/jq" + exit 1 +fi + +PR_NUMBER="$1" +PATCH_FILE="$2" +TMP_BRANCH="$3" +if [ -z "$PR_NUMBER" ]; then + echo "${LOGPREFIX} Please provide a PR link or number. For example: https://github.com/ossrs/ffmpeg-webrtc/pull/20" + exit 1 +fi + +if [[ "$1" =~ ^https://github.com/ossrs/ffmpeg-webrtc/pull/([0-9]+)$ ]]; then + PR_NUMBER="${BASH_REMATCH[1]}" +elif [[ "$1" =~ ^[0-9]+$ ]]; then + PR_NUMBER="$1" +else + echo "${LOGPREFIX} Invalid input format. Please provide a PR link or number. For example: https://github.com/ossrs/ffmpeg-webrtc/pull/20" + exit 1 +fi + +PR_URL="https://github.com/ossrs/ffmpeg-webrtc/pull/$PR_NUMBER" +echo "${LOGPREFIX} Fetching PR #$PR_NUMBER from $PR_URL" + +PR_DATA=$(curl -s "https://api.github.com/repos/ossrs/ffmpeg-webrtc/pulls/$PR_NUMBER") +REPO_NAME=$(printf '%s' "$PR_DATA" | jq -r '.head.repo.full_name') +BRANCH_NAME=$(printf '%s' "$PR_DATA" | jq -r '.head.ref') +echo "${LOGPREFIX} Repository: $REPO_NAME, Branch: $BRANCH_NAME" +if [[ -z "$REPO_NAME" || -z "$BRANCH_NAME" ]]; then + echo "${LOGPREFIX} Error: REPO_NAME or BRANCH_NAME is empty!" + exit 1 +fi + +PR_TITLE=$(printf '%s' "$PR_DATA" | jq -r '.title') +PR_DESCRIPTION=$(printf '%s' "$PR_DATA" | jq -r '.body // ""') +echo "${LOGPREFIX} PR information:" +echo "${LOGPREFIX} ===================================================================" +echo "${LOGPREFIX} $PR_TITLE" +echo "${LOGPREFIX} $PR_DESCRIPTION" +echo "${LOGPREFIX} ===================================================================" +echo "${LOGPREFIX} " +if [[ -z "$PR_TITLE" ]]; then + echo "${LOGPREFIX} Error: PR title is empty!" + exit 1 +fi + +git checkout workflows && +echo "${LOGPREFIX} Switched to workflows branch." && +git pull && +echo "${LOGPREFIX} Pulled latest changes from workflows branch." +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to switch to workflows branch or pull latest changes." + exit 1 +fi + +REMOTE_NAME=patch-tmp && +if git remote | grep -q "^$REMOTE_NAME$"; then + git remote rm "$REMOTE_NAME" +fi && +git remote add $REMOTE_NAME https://github.com/${REPO_NAME}.git && +git fetch $REMOTE_NAME $BRANCH_NAME && +echo "${LOGPREFIX} Fetch remote $REMOTE_NAME at $(git remote get-url $REMOTE_NAME)" +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to fetch remote branch $BRANCH_NAME from $REMOTE_NAME." + exit 1 +fi + +if [[ -z "$TMP_BRANCH" ]]; then + TMP_BRANCH="tmp-branch-for-patch-$PR_NUMBER" +fi && +if git branch --list "$TMP_BRANCH" | grep -q "^..$TMP_BRANCH$"; then + git branch -D "$TMP_BRANCH" +fi && +git checkout -b $TMP_BRANCH $REMOTE_NAME/$BRANCH_NAME && +echo "${LOGPREFIX} Checkout branch $TMP_BRANCH from $REMOTE_NAME/$BRANCH_NAME" +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to checkout branch $TMP_BRANCH from $REMOTE_NAME/$BRANCH_NAME." + exit 1 +fi + +FIRST_AUTHOR_NAME=$(git log workflows..HEAD --reverse --format='%an' | head -n1) +FIRST_AUTHOR_EMAIL=$(git log workflows..HEAD --reverse --format='%ae' | head -n1) +echo "${LOGPREFIX} Author: $FIRST_AUTHOR_NAME <$FIRST_AUTHOR_EMAIL>" +if [[ -z "$FIRST_AUTHOR_NAME" || -z "$FIRST_AUTHOR_EMAIL" ]]; then + echo "${LOGPREFIX} Error: Unable to determine the first author of the PR." + exit 1 +fi + +COAUTHORS=$(git log workflows..HEAD --format='Co-authored-by: %an <%ae>' |grep -v "$FIRST_AUTHOR_NAME" | sort -u) +COAUTHOR_COUNT=$(echo "$COAUTHORS" | wc -l) +if [[ "$COAUTHOR_COUNT" -gt 0 ]]; then + echo "${LOGPREFIX} $COAUTHORS" +fi + +COMMIT_MSG="$PR_TITLE" +if [[ -n "$PR_DESCRIPTION" ]]; then + COMMIT_MSG="$COMMIT_MSG\n\n$PR_DESCRIPTION" +fi + +if [[ "$COAUTHOR_COUNT" -gt 0 ]]; then + COMMIT_MSG="$COMMIT_MSG\n" + COMMIT_MSG="$COMMIT_MSG\n$COAUTHORS" +fi + +echo "${LOGPREFIX} Commit information:" +echo "${LOGPREFIX} Author: $FIRST_AUTHOR_NAME <$FIRST_AUTHOR_EMAIL>" +echo "${LOGPREFIX} ===================================================================" +echo -e "$COMMIT_MSG" +echo "${LOGPREFIX} ===================================================================" +echo "${LOGPREFIX} " + +if [[ $(git config --list --local |grep 'user.name' >/dev/null 2>&1 && echo yes) != "yes" ]]; then + git config --local user.name "$FIRST_AUTHOR_NAME" +fi && +if [[ $(git config --list --local |grep 'user.email' >/dev/null 2>&1 && echo yes) != "yes" ]]; then + git config --local user.email "$FIRST_AUTHOR_EMAIL" +fi && +git config --list && +echo "${LOGPREFIX} Set local git user configuration to: $FIRST_AUTHOR_NAME <$FIRST_AUTHOR_EMAIL>" +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to set local git user configuration." + exit 1 +fi + +git rebase workflows && +git reset --soft workflows && +echo "${LOGPREFIX} Rebased onto workflows branch and reset to soft." +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to rebase or reset changes." + exit 1 +fi + +git status && +git restore --staged .github && +git restore .github && +git status && +echo "${LOGPREFIX} Restored .github directory to the state of workflows branch." +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to restore .github directory." + exit 1 +fi + +if [[ $(git status | grep 'nothing to commit, working tree clean' >/dev/null 2>&1 && echo yes) == "yes" ]]; then + echo "${LOGPREFIX} No changes to commit. Exiting." + git checkout workflows + exit 0 +fi + +git commit --author "$FIRST_AUTHOR_NAME <$FIRST_AUTHOR_EMAIL>" -m "$(echo -e "$COMMIT_MSG")" && +echo "${LOGPREFIX} Squashed commits into a single commit." +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to rebase or commit changes." + exit 1 +fi + +git branch -vv && +git log -1 --pretty=format:"%an <%ae> %h %s" +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to display branch information or last commit." + exit 1 +fi + +if [[ -z "$PATCH_FILE" ]]; then + PATCH_FILE="whip-patch-$PR_NUMBER-$(date +%s).patch" +fi && +rm -f $PATCH_FILE && +git format-patch --add-header "X-Unsent: 1" --to ffmpeg-devel@ffmpeg.org -1 --stdout > $PATCH_FILE && +echo "${LOGPREFIX} Created patch file: $PATCH_FILE" +if [[ $? -ne 0 ]]; then + echo "${LOGPREFIX} Failed to create patch file." + exit 1 +fi + +git checkout workflows +#git br -D $TMP_BRANCH +#echo "${LOGPREFIX} Removed temporary branch $TMP_BRANCH." + +echo "${LOGPREFIX} " +echo "${LOGPREFIX} Patch file created: $PATCH_FILE" +echo "${LOGPREFIX} " diff --git a/.github/workflows/fate-cache.yml b/.github/workflows/fate-cache.yml new file mode 100644 index 0000000000000..77ea32e3f3f4e --- /dev/null +++ b/.github/workflows/fate-cache.yml @@ -0,0 +1,27 @@ +name: "FFmpeg FATE Cache" + +on: + workflow_dispatch: + +permissions: read-all + +jobs: + build: + name: "Build FFmpeg Fate Cache" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Login to docker hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: "${{ secrets.DOCKER_USERNAME }}" + password: "${{ secrets.DOCKER_PASSWORD }}" + - name: Build FFmpeg Fate Cache + run: | + set -euxo pipefail + docker build -t ossrs/srs:ffmpeg-fate -f .github/docker/Dockerfile . + - name: Push FFmpeg Fate Cache + run: | + set -euxo pipefail + docker push ossrs/srs:ffmpeg-fate + runs-on: ubuntu-22.04 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000000..1a8ced2650028 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,847 @@ +name: "Test" + +on: + push: + pull_request: + +permissions: read-all + +# Results for commonly used commands: +# $HOME is /home/runner +# $(pwd) is /home/runner/work/ffmpeg-webrtc/ffmpeg-webrtc +# $(nproc) is 4 +# $(whoami) is runner +# $(id -gn) is docker +# $(which docker) is /usr/bin/docker +# $(ifconfig eth0 | grep 'inet ' | awk '{print $2}') is private IP4 address like 10.1.0.76 +jobs: + build: + name: "Build FFmpeg" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config libssl-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + runs-on: ubuntu-22.04 + + fate: + name: "FFmpeg Fate Test" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # The cache for FFmpeg FATE samples can help decrease the resync time when executing + # "make fate-resync." The cache is stored in the Docker image "ossrs/srs:ffmpeg-fate," + # which can be refreshed by manually executing the below workflow. + # https://github.com/ossrs/ffmpeg-webrtc/actions/workflows/fate-cache.yml + - name: Download Fate Cache Samples + run: | + set -euxo pipefail + + docker run --rm -v $(pwd):/target ossrs/srs:ffmpeg-fate \ + bash -c "cp /opt/ffmpeg/fate-suite.tar /target/" + tar xf fate-suite.tar + + ls -ldh fate-suite + du -sh fate-suite + - name: Configure FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config libssl-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --extra-cflags='-fsanitize=address -g -O0' --extra-cxxflags='-fsanitize=address -g -O0' --extra-ldflags='-fsanitize=address -g -O0' + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: FFmpeg Fate rsync + run: | + set -euxo pipefail + make fate-rsync SAMPLES=$(pwd)/fate-suite + - name: Stat Fate Suite + run: | + set -euxo pipefail + du -sh fate-suite + du -sh * + - name: Run FFmpeg Fate + run: | + set -euxo pipefail + make fate -j$(nproc) SAMPLES=$(pwd)/fate-suite + runs-on: ubuntu-22.04 + + srs: + name: "FFmpeg with SRS" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libssl-dev libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + pion: + name: "FFmpeg with Pion" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libssl-dev libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.22' + - name: Verify Go version + run: go version + - name: Start Pion + run: | + set -euxo pipefail + git clone https://github.com/pion/webrtc.git + cd webrtc/examples/whip-whep + go run *.go & + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip -authorization "seanTest" "http://localhost:8080/whip" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + janus: + name: "FFmpeg with Janus" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libssl-dev libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start Janus + run: | + set -euxo pipefail + git clone https://github.com/winlinvip/janus-docker.git + (cd janus-docker && + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') && + sed -i "s|\(^[[:blank:]]*nat_1_1_mapping *=\).*|\1\"$ip\"|g" janus.jcfg && + docker run --rm -d -p 8081:8080 -p 8188:8188 -p 8443:8443 -p 20000-20010:20000-20010/udp \ + -v $(pwd)/janus.jcfg:/usr/local/etc/janus/janus.jcfg \ + -v $(pwd)/janus.plugin.videoroom.jcfg:/usr/local/etc/janus/janus.plugin.videoroom.jcfg \ + -v $(pwd)/janus.transport.http.jcfg:/usr/local/etc/janus/janus.transport.http.jcfg \ + -v $(pwd)/janus.transport.websockets.jcfg:/usr/local/etc/janus/janus.transport.websockets.jcfg \ + -v $(pwd)/videoroomtest.js:/usr/local/share/janus/demos/videoroomtest.js \ + ossrs/janus:v1.0.12) + + git clone https://github.com/meetecho/simple-whip-server.git + cd simple-whip-server + git checkout bd2d98898b9842bfc329443b46bcc906aab857aa + npm install + npm run build + npm run start & + + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + curl -H 'Content-Type: application/json' -d '{"id": "abc123", "room": 2345}' \ + http://localhost:7080/whip/create + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip 'http://localhost:7080/whip/endpoint/abc123' \ + 1>ffstdout.log 2>ffstderr.log & + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + runs-on: ubuntu-22.04 + + asan: + name: "FFmpeg with Asan" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libssl-dev libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus \ + --extra-cflags='-fsanitize=address -g -O0' --extra-cxxflags='-fsanitize=address -g -O0' --extra-ldflags='-fsanitize=address -g -O0' + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + # TEST: Generate a coredump. + #pkill -SIGSEGV ffmpeg && sleep 3 && exit 0 + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check Asan Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'ERROR: AddressSanitizer' && + echo "AddressSanitizer error found in ffstderr.log" && exit 1 + echo "AddressSanitizer is ok" + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + openssl-1-0-1k: + name: "With OpenSSL 1.0.1k" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build OpenSSL 1.0.1k + run: | + set -euxo pipefail + curl -s -L https://www.openssl.org/source/openssl-1.0.1k.tar.gz | tar xz + cd openssl-1.0.1k + ./config --prefix=$HOME/.release/openssl && make -j1 + sudo make install_sw + - name: Download Test File + run: | + set -euxo pipefail + curl -s -L -O https://github.com/ossrs/ffmpeg-webrtc/releases/download/pre-release/bbb-4mbps-baseline-opus.mp4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + PKG_CONFIG_PATH="$HOME/.release/openssl/lib/pkgconfig" \ + ./configure --enable-muxer=whip --enable-openssl --enable-version3 + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -i bbb-4mbps-baseline-opus.mp4 -c copy \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + openssl-1-0-2: + name: "With OpenSSL 1.0.2" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build OpenSSL 1.0.2 + run: | + set -euxo pipefail + curl -s -L https://www.openssl.org/source/openssl-1.0.2.tar.gz | tar xz + cd openssl-1.0.2 + ./config --prefix=$HOME/.release/openssl + make -j1 && sudo make install_sw + - name: Download Test File + run: | + set -euxo pipefail + curl -s -L -O https://github.com/ossrs/ffmpeg-webrtc/releases/download/pre-release/bbb-4mbps-baseline-opus.mp4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + PKG_CONFIG_PATH="$HOME/.release/openssl/lib/pkgconfig" \ + ./configure --enable-muxer=whip --enable-openssl --enable-version3 + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -i bbb-4mbps-baseline-opus.mp4 -c copy \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + openssl-1-1-0h: + name: "With OpenSSL 1.1.0h" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build OpenSSL 1.1.0h + run: | + set -euxo pipefail + curl -s -L https://www.openssl.org/source/openssl-1.1.0h.tar.gz | tar xz + cd openssl-1.1.0h + ./config --prefix=$HOME/.release/openssl + make -j$(nproc) && sudo make install_sw + - name: Download Test File + run: | + set -euxo pipefail + curl -s -L -O https://github.com/ossrs/ffmpeg-webrtc/releases/download/pre-release/bbb-4mbps-baseline-opus.mp4 + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + PKG_CONFIG_PATH="$HOME/.release/openssl/lib/pkgconfig" \ + ./configure --enable-muxer=whip --enable-openssl --enable-version3 + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -i bbb-4mbps-baseline-opus.mp4 -c copy \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + openssl-3-0: + name: "With OpenSSL 3.0" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build OpenSSL 3.0 + run: | + set -euxo pipefail + curl -s -L https://www.openssl.org/source/openssl-3.0.0.tar.gz | tar xz + cd openssl-3.0.0 + ./config --prefix=$HOME/.release/openssl + make -j$(nproc) && sudo make install_sw + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + PKG_CONFIG_PATH="$HOME/.release/openssl/lib/pkgconfig" \ + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + openssl-latest: + name: "With OpenSSL latest" + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build OpenSSL latest + run: | + set -euxo pipefail + curl -s -L https://www.openssl.org/source/openssl-3.5.0.tar.gz | tar xz + cd openssl-3.5.0 + ./config --prefix=$HOME/.release/openssl + make -j$(nproc) && sudo make install_sw + - name: Build FFmpeg + run: | + set -euxo pipefail + + # Install dependencies + sudo apt-get update + sudo apt-get install -y nasm pkg-config jq libopus-dev libx264-dev + + # Build FFmpeg with WebRTC support + PKG_CONFIG_PATH="$HOME/.release/openssl/lib/pkgconfig" \ + ./configure --enable-muxer=whip --enable-openssl --enable-version3 \ + --enable-libx264 --enable-gpl --enable-libopus + make -j$(nproc) + ./ffmpeg -version && ./ffmpeg -muxers 2>/dev/null |grep whip + - name: Start SRS Docker container + run: | + set -euxo pipefail + ip=$(ifconfig eth0 | grep 'inet ' | awk '{print $2}') + docker run --rm -d -p 1935:1935 -p 1985:1985 -p 8080:8080 \ + --env CANDIDATE=$ip -p 8000:8000/udp \ + ossrs/srs:5 ./objs/srs -c conf/rtc2rtmp.conf + - name: Streaming with FFmpeg + run: | + set -euxo pipefail + nohup ./ffmpeg -t 30 -re -f lavfi -i testsrc=size=1280x720 -f lavfi -i sine=frequency=440 -pix_fmt yuv420p \ + -vcodec libx264 -profile:v baseline -r 25 -g 50 -acodec libopus -ar 48000 -ac 2 \ + -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream" \ + 1>ffstdout.log 2>ffstderr.log & + - name: Check SRS Streaming + id: streaming + run: | + set -euxo pipefail + + # Check streams in SRS. + for ((i=0; i<10; i++)); do + STREAM=$(curl -s http://localhost:1985/api/v1/streams/ | jq -r '.streams[].name') + if [[ "$STREAM" == "livestream" ]]; then + echo 'Test OK'; + echo "has_stream=true" >> $GITHUB_OUTPUT + break; + fi + sleep 3 + done + + if [[ "$STREAM" != "livestream" ]]; then + echo "Stream not found: $STREAM" + echo "has_stream=false" >> $GITHUB_OUTPUT + fi + - name: Stop FFmpeg normally + run: | + pkill -SIGINT ffmpeg && sleep 3 || + echo "FFmpeg process not found or already stopped." + - name: Show FFmpeg Stdout Log + run: cat ffstdout.log + - name: Show FFmpeg Stderr Log + run: cat ffstderr.log + - name: Check FFmpeg Exit Log + run: | + set -euxo pipefail + cat ffstderr.log |grep 'Exiting normally' && exit 0 + echo "Exiting normally not found in ffstderr.log" && exit 1 + - name: Check Stream Existence + if: ${{ steps.streaming.outputs.has_stream == 'false' }} + run: exit 1 + runs-on: ubuntu-22.04 + + generate-patch: + name: "Generate Patch" + if: ${{ github.event_name == 'pull_request' }} + steps: + # Checkout to workflows branch, make sure the base branch is available. + - name: Checkout repository with workflows branch + uses: actions/checkout@v4 + with: + ref: workflows + fetch-depth: 0 + - name: Try to checkout to workflows branch + run: | + set -euxo pipefail + git checkout workflows + git branch -vv + # Checkout to PR commit, use the lastest script. + - name: Checkout repository to PR commit + uses: actions/checkout@v4 + - name: Show Git Info + run: | + set -euxo pipefail + git branch -vv + echo "Repository: ${{ github.repository }}" + echo "Ref: ${{ github.ref }}" + echo "Event Name: ${{ github.event_name }}" + echo "Pull Request Number: ${{ github.event.pull_request.number }}" + - name: Install Dependencies + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y jq + - name: Run Script + id: format_patch + run: | + set -euxo pipefail + + PR_NUMBER=${{ github.event.pull_request.number }} + PATCH_FILENAME="whip-patch-$PR_NUMBER-$(date +%s)" + TMP_BRANCH="tmp-branch-for-patch-$PR_NUMBER" + echo "PR ID is ${{ github.event.pull_request.number }}" + echo "Patch file is $PATCH_FILENAME.patch" + echo "Temporary branch is $TMP_BRANCH" + + bash .github/scripts/format-patch.sh $PR_NUMBER "$PATCH_FILENAME.patch" + echo "patch_file=$PATCH_FILENAME" >> $GITHUB_OUTPUT + echo "temporary_branch=$TMP_BRANCH" >> $GITHUB_OUTPUT + + if [[ -f "$PATCH_FILENAME.patch" ]]; then + echo "has_patch=true" >> $GITHUB_OUTPUT + else + echo "has_patch=false" >> $GITHUB_OUTPUT + fi + - name: Show Branch Info + if: ${{ steps.format_patch.outputs.has_patch == 'true' }} + run: git show ${{ steps.format_patch.outputs.temporary_branch }} + - name: Show Patch File + if: ${{ steps.format_patch.outputs.has_patch == 'true' }} + run: cat ${{ steps.format_patch.outputs.patch_file }}.patch + - name: Upload all patch files + if: ${{ steps.format_patch.outputs.has_patch == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.format_patch.outputs.patch_file }} + path: | + whip-*.patch + retention-days: 90 + runs-on: ubuntu-22.04 + + test-done: + needs: + - fate + - srs + - pion + - janus + - asan + - openssl-1-0-1k + - openssl-1-0-2 + - openssl-1-1-0h + - openssl-3-0 + - openssl-latest + - generate-patch + steps: + - run: echo 'All done' + runs-on: ubuntu-22.04 + diff --git a/.gitignore b/.gitignore index 59c89da5e03b5..9e56f800f8c67 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ *.ptx *.ptx.c *.ptx.gz +*.patch *_g \#* .\#* @@ -45,3 +46,4 @@ /libavcodec/vulkan/*.c /libavfilter/vulkan/*.c /.*/ +/fate-suite diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c index 2a3905891d197..4733faec9c61e 100644 --- a/libavformat/tls_openssl.c +++ b/libavformat/tls_openssl.c @@ -316,7 +316,8 @@ static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerpri goto enomem_end; } - serial = (int)av_get_random_seed(); + // According to RFC5280 4.1.2.2, The serial number MUST be a positive integer + serial = (int)(av_get_random_seed() & 0x7FFFFFFF); if (ASN1_INTEGER_set(X509_get_serialNumber(*cert), serial) != 1) { av_log(NULL, AV_LOG_ERROR, "TLS: Failed to set serial, %s\n", ERR_error_string(ERR_get_error(), NULL)); goto einval_end; diff --git a/libavformat/whip.c b/libavformat/whip.c index 5fdbd6949d541..18fc97f57ccf6 100644 --- a/libavformat/whip.c +++ b/libavformat/whip.c @@ -114,6 +114,7 @@ /* Referring to Chrome's definition of RTP payload types. */ #define WHIP_RTP_PAYLOAD_TYPE_H264 106 #define WHIP_RTP_PAYLOAD_TYPE_OPUS 111 +#define WHIP_RTP_PAYLOAD_TYPE_RTX 105 /** * The STUN message header, which is 20 bytes long, comprises the @@ -150,6 +151,11 @@ #define WHIP_SDP_SESSION_ID "4489045141692799359" #define WHIP_SDP_CREATOR_IP "127.0.0.1" +/** + * Retransmission / NACK support +*/ +#define HISTORY_SIZE_DEFAULT 512 + /* Calculate the elapsed time from starttime to endtime in milliseconds. */ #define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000) @@ -193,9 +199,24 @@ enum WHIPState { WHIP_STATE_FAILED, }; +typedef enum WHIPFlags { + WHIP_FLAG_IGNORE_IPV6 = (1 << 0), // Ignore ipv6 candidate + WHIP_FLAG_DISABLE_RTX = (1 << 1) // Enable NACK and RTX +} WHIPFlags; + +typedef struct RtpHistoryItem { + /* original RTP seq */ + uint16_t seq; + /* length in bytes */ + int size; + /* malloc-ed copy */ + uint8_t* buf; +} RtpHistoryItem; + typedef struct WHIPContext { AVClass *av_class; + uint32_t flags; // enum WHIPFlags /* The state of the RTC connection. */ enum WHIPState state; /* The callback return value for DTLS. */ @@ -205,6 +226,7 @@ typedef struct WHIPContext { /* Parameters for the input audio and video codecs. */ AVCodecParameters *audio_par; AVCodecParameters *video_par; + uint8_t constraint_set_flags; /** * The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet; @@ -279,6 +301,7 @@ typedef struct WHIPContext { /* The SRTP send context, to encrypt outgoing packets. */ SRTPContext srtp_audio_send; SRTPContext srtp_video_send; + SRTPContext srtp_video_rtx_send; SRTPContext srtp_rtcp_send; /* The SRTP receive context, to decrypt incoming packets. */ SRTPContext srtp_recv; @@ -303,6 +326,14 @@ typedef struct WHIPContext { /* The certificate and private key used for DTLS handshake. */ char* cert_file; char* key_file; + + /* RTX / NACK */ + uint8_t rtx_payload_type; + uint32_t video_rtx_ssrc; + uint16_t rtx_seq; + int history_size; + RtpHistoryItem *history; /* ring buffer */ + int hist_head; } WHIPContext; /** @@ -359,14 +390,14 @@ static int dtls_context_on_state(AVFormatContext *s, const char* type, const cha if (state == DTLS_STATE_CLOSED) { whip->dtls_closed = 1; - av_log(whip, AV_LOG_VERBOSE, "WHIP: DTLS session closed, type=%s, desc=%s, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "DTLS session closed, type=%s, desc=%s, elapsed=%dms\n", type ? type : "", desc ? desc : "", ELAPSED(whip->whip_starttime, av_gettime())); goto error; } if (state == DTLS_STATE_FAILED) { whip->state = WHIP_STATE_FAILED; - av_log(whip, AV_LOG_ERROR, "WHIP: DTLS session failed, type=%s, desc=%s\n", + av_log(whip, AV_LOG_ERROR, "DTLS session failed, type=%s, desc=%s\n", type ? type : "", desc ? desc : ""); whip->dtls_ret = AVERROR(EIO); goto error; @@ -375,7 +406,7 @@ static int dtls_context_on_state(AVFormatContext *s, const char* type, const cha if (state == DTLS_STATE_FINISHED && whip->state < WHIP_STATE_DTLS_FINISHED) { whip->state = WHIP_STATE_DTLS_FINISHED; whip->whip_dtls_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: DTLS handshake is done, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n", ELAPSED(whip->whip_starttime, av_gettime())); return ret; } @@ -404,7 +435,7 @@ static av_cold int initialize(AVFormatContext *s) ret = certificate_key_init(s); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to init certificate and key\n"); + av_log(whip, AV_LOG_ERROR, "Failed to init certificate and key\n"); return ret; } @@ -413,13 +444,13 @@ static av_cold int initialize(AVFormatContext *s) av_lfg_init(&whip->rnd, seed); if (whip->pkt_size < ideal_pkt_size) - av_log(whip, AV_LOG_WARNING, "WHIP: pkt_size=%d(<%d) is too small, may cause packet loss\n", + av_log(whip, AV_LOG_WARNING, "pkt_size=%d(<%d) is too small, may cause packet loss\n", whip->pkt_size, ideal_pkt_size); if (whip->state < WHIP_STATE_INIT) whip->state = WHIP_STATE_INIT; whip->whip_init_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: Init state=%d, handshake_timeout=%dms, pkt_size=%d, seed=%d, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "Init state=%d, handshake_timeout=%dms, pkt_size=%d, seed=%d, elapsed=%dms\n", whip->state, whip->handshake_timeout, whip->pkt_size, seed, ELAPSED(whip->whip_starttime, av_gettime())); return 0; @@ -440,45 +471,30 @@ static av_cold int initialize(AVFormatContext *s) static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par) { int ret = 0; - const uint8_t *r = par->extradata, *r1, *end = par->extradata + par->extradata_size; - H264SPS seq, *const sps = &seq; - uint32_t state; + const uint8_t *r = par->extradata; WHIPContext *whip = s->priv_data; if (par->codec_id != AV_CODEC_ID_H264) return ret; - if (par->profile != AV_PROFILE_UNKNOWN && par->level != AV_LEVEL_UNKNOWN) - return ret; - if (!par->extradata || par->extradata_size <= 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unable to parse profile from empty extradata=%p, size=%d\n", + av_log(whip, AV_LOG_ERROR, "Unable to parse profile from empty extradata=%p, size=%d\n", par->extradata, par->extradata_size); return AVERROR(EINVAL); } - while (1) { - r = avpriv_find_start_code(r, end, &state); - if (r >= end) - break; - - r1 = ff_nal_find_startcode(r, end); - if ((state & 0x1f) == H264_NAL_SPS) { - ret = ff_avc_decode_sps(sps, r, r1 - r); - if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to decode SPS, state=%x, size=%d\n", - state, (int)(r1 - r)); - return ret; - } - - av_log(whip, AV_LOG_VERBOSE, "WHIP: Parse profile=%d, level=%d from SPS\n", - sps->profile_idc, sps->level_idc); - par->profile = sps->profile_idc; - par->level = sps->level_idc; - } + if (AV_RB32(r) == 0x00000001 && (r[4] & 0x1F) == 7) + r = &r[5]; + else if (AV_RB24(r) == 0x000001 && (r[3] & 0x1F) == 7) + r = &r[4]; + else if (r[0] == 0x01) // avcC + r = &r[1]; + else + return AVERROR(EINVAL); - r = r1; - } + if (par->profile == AV_PROFILE_UNKNOWN) par->profile = r[0]; + whip->constraint_set_flags = r[1]; + if (par->level == AV_LEVEL_UNKNOWN) par->level = r[2]; return ret; } @@ -515,62 +531,62 @@ static int parse_codec(AVFormatContext *s) switch (par->codec_type) { case AVMEDIA_TYPE_VIDEO: if (whip->video_par) { - av_log(whip, AV_LOG_ERROR, "WHIP: Only one video stream is supported by RTC\n"); + av_log(whip, AV_LOG_ERROR, "Only one video stream is supported by RTC\n"); return AVERROR(EINVAL); } whip->video_par = par; if (par->codec_id != AV_CODEC_ID_H264) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unsupported video codec %s by RTC, choose h264\n", + av_log(whip, AV_LOG_ERROR, "Unsupported video codec %s by RTC, choose h264\n", desc ? desc->name : "unknown"); return AVERROR_PATCHWELCOME; } if (par->video_delay > 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unsupported B frames by RTC\n"); + av_log(whip, AV_LOG_ERROR, "Unsupported B frames by RTC\n"); return AVERROR_PATCHWELCOME; } if ((ret = parse_profile_level(s, par)) < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to parse SPS/PPS from extradata\n"); + av_log(whip, AV_LOG_ERROR, "Failed to parse SPS/PPS from extradata\n"); return AVERROR(EINVAL); } if (par->profile == AV_PROFILE_UNKNOWN) { - av_log(whip, AV_LOG_WARNING, "WHIP: No profile found in extradata, consider baseline\n"); + av_log(whip, AV_LOG_WARNING, "No profile found in extradata, consider baseline\n"); return AVERROR(EINVAL); } if (par->level == AV_LEVEL_UNKNOWN) { - av_log(whip, AV_LOG_WARNING, "WHIP: No level found in extradata, consider 3.1\n"); + av_log(whip, AV_LOG_WARNING, "No level found in extradata, consider 3.1\n"); return AVERROR(EINVAL); } break; case AVMEDIA_TYPE_AUDIO: if (whip->audio_par) { - av_log(whip, AV_LOG_ERROR, "WHIP: Only one audio stream is supported by RTC\n"); + av_log(whip, AV_LOG_ERROR, "Only one audio stream is supported by RTC\n"); return AVERROR(EINVAL); } whip->audio_par = par; if (par->codec_id != AV_CODEC_ID_OPUS) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unsupported audio codec %s by RTC, choose opus\n", + av_log(whip, AV_LOG_ERROR, "Unsupported audio codec %s by RTC, choose opus\n", desc ? desc->name : "unknown"); return AVERROR_PATCHWELCOME; } if (par->ch_layout.nb_channels != 2) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unsupported audio channels %d by RTC, choose stereo\n", + av_log(whip, AV_LOG_ERROR, "Unsupported audio channels %d by RTC, choose stereo\n", par->ch_layout.nb_channels); return AVERROR_PATCHWELCOME; } if (par->sample_rate != 48000) { - av_log(whip, AV_LOG_ERROR, "WHIP: Unsupported audio sample rate %d by RTC, choose 48000\n", par->sample_rate); + av_log(whip, AV_LOG_ERROR, "Unsupported audio sample rate %d by RTC, choose 48000\n", par->sample_rate); return AVERROR_PATCHWELCOME; } break; default: - av_log(whip, AV_LOG_ERROR, "WHIP: Codec type '%s' for stream %d is not supported by RTC\n", + av_log(whip, AV_LOG_ERROR, "Codec type '%s' for stream %d is not supported by RTC\n", av_get_media_type_string(par->codec_type), i); return AVERROR_PATCHWELCOME; } @@ -589,7 +605,7 @@ static int parse_codec(AVFormatContext *s) */ static int generate_sdp_offer(AVFormatContext *s) { - int ret = 0, profile, level, profile_iop; + int ret = 0, profile, level; const char *acodec_name = NULL, *vcodec_name = NULL; AVBPrint bp; WHIPContext *whip = s->priv_data; @@ -598,7 +614,7 @@ static int generate_sdp_offer(AVFormatContext *s) av_bprint_init(&bp, 1, MAX_SDP_SIZE); if (whip->sdp_offer) { - av_log(whip, AV_LOG_ERROR, "WHIP: SDP offer is already set\n"); + av_log(whip, AV_LOG_ERROR, "SDP offer is already set\n"); ret = AVERROR(EINVAL); goto end; } @@ -615,6 +631,16 @@ static int generate_sdp_offer(AVFormatContext *s) whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS; whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264; + /* RTX and NACK init */ + whip->rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_RTX; + whip->video_rtx_ssrc = av_lfg_get(&whip->rnd); + whip->rtx_seq = 0; + whip->hist_head = 0; + whip->history_size = FFMAX(64, whip->history_size); + whip->history = av_calloc(whip->history_size, sizeof(*whip->history)); + if (!whip->history) + return AVERROR(ENOMEM); + av_bprintf(&bp, "" "v=0\r\n" "o=FFmpeg %s 2 IN IP4 %s\r\n" @@ -657,16 +683,15 @@ static int generate_sdp_offer(AVFormatContext *s) } if (whip->video_par) { - profile_iop = profile = whip->video_par->profile; + profile = whip->video_par->profile; level = whip->video_par->level; if (whip->video_par->codec_id == AV_CODEC_ID_H264) { vcodec_name = "H264"; - profile_iop &= AV_PROFILE_H264_CONSTRAINED; profile &= (~AV_PROFILE_H264_CONSTRAINED); } av_bprintf(&bp, "" - "m=video 9 UDP/TLS/RTP/SAVPF %u\r\n" + "m=video 9 UDP/TLS/RTP/SAVPF %u %u\r\n" "c=IN IP4 0.0.0.0\r\n" "a=ice-ufrag:%s\r\n" "a=ice-pwd:%s\r\n" @@ -679,9 +704,16 @@ static int generate_sdp_offer(AVFormatContext *s) "a=rtcp-rsize\r\n" "a=rtpmap:%u %s/90000\r\n" "a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n" + "a=rtcp-fb:%u nack\r\n" + "a=rtpmap:%u rtx/90000\r\n" + "a=fmtp:%u apt=%u\r\n" + "a=ssrc-group:FID %u %u\r\n" + "a=ssrc:%u cname:FFmpeg\r\n" + "a=ssrc:%u msid:FFmpeg video\r\n" "a=ssrc:%u cname:FFmpeg\r\n" "a=ssrc:%u msid:FFmpeg video\r\n", whip->video_payload_type, + whip->rtx_payload_type, whip->ice_ufrag_local, whip->ice_pwd_local, whip->dtls_fingerprint, @@ -689,14 +721,22 @@ static int generate_sdp_offer(AVFormatContext *s) vcodec_name, whip->video_payload_type, profile, - profile_iop, + whip->constraint_set_flags, level, + whip->video_payload_type, + whip->rtx_payload_type, + whip->rtx_payload_type, + whip->video_payload_type, whip->video_ssrc, - whip->video_ssrc); + whip->video_rtx_ssrc, + whip->video_ssrc, + whip->video_ssrc, + whip->video_rtx_ssrc, + whip->video_rtx_ssrc); } if (!av_bprint_is_complete(&bp)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Offer exceed max %d, %s\n", MAX_SDP_SIZE, bp.str); + av_log(whip, AV_LOG_ERROR, "Offer exceed max %d, %s\n", MAX_SDP_SIZE, bp.str); ret = AVERROR(EIO); goto end; } @@ -710,7 +750,7 @@ static int generate_sdp_offer(AVFormatContext *s) if (whip->state < WHIP_STATE_OFFER) whip->state = WHIP_STATE_OFFER; whip->whip_offer_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: Generated state=%d, offer: %s\n", whip->state, whip->sdp_offer); + av_log(whip, AV_LOG_VERBOSE, "Generated state=%d, offer: %s\n", whip->state, whip->sdp_offer); end: av_bprint_finalize(&bp, NULL); @@ -745,7 +785,7 @@ static int exchange_sdp(AVFormatContext *s) } if (!whip->sdp_offer || !strlen(whip->sdp_offer)) { - av_log(whip, AV_LOG_ERROR, "WHIP: No offer to exchange\n"); + av_log(whip, AV_LOG_ERROR, "No offer to exchange\n"); ret = AVERROR(EINVAL); goto end; } @@ -754,7 +794,7 @@ static int exchange_sdp(AVFormatContext *s) if (whip->authorization) ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization); if (ret <= 0 || ret >= sizeof(buf)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to generate headers, size=%d, %s\n", ret, buf); + av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); ret = AVERROR(EINVAL); goto end; } @@ -773,7 +813,7 @@ static int exchange_sdp(AVFormatContext *s) ret = ffurl_open_whitelist(&whip_uc, s->url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to request url=%s, offer: %s\n", s->url, whip->sdp_offer); + av_log(whip, AV_LOG_ERROR, "Failed to request url=%s, offer: %s\n", s->url, whip->sdp_offer); goto end; } @@ -793,21 +833,21 @@ static int exchange_sdp(AVFormatContext *s) break; } if (ret <= 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read response from url=%s, offer is %s, answer is %s\n", + av_log(whip, AV_LOG_ERROR, "Failed to read response from url=%s, offer is %s, answer is %s\n", s->url, whip->sdp_offer, whip->sdp_answer); goto end; } av_bprintf(&bp, "%.*s", ret, buf); if (!av_bprint_is_complete(&bp)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Answer exceed max size %d, %.*s, %s\n", MAX_SDP_SIZE, ret, buf, bp.str); + av_log(whip, AV_LOG_ERROR, "Answer exceed max size %d, %.*s, %s\n", MAX_SDP_SIZE, ret, buf, bp.str); ret = AVERROR(EIO); goto end; } } if (!av_strstart(bp.str, "v=", NULL)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Invalid answer: %s\n", bp.str); + av_log(whip, AV_LOG_ERROR, "Invalid answer: %s\n", bp.str); ret = AVERROR(EINVAL); goto end; } @@ -820,7 +860,7 @@ static int exchange_sdp(AVFormatContext *s) if (whip->state < WHIP_STATE_ANSWER) whip->state = WHIP_STATE_ANSWER; - av_log(whip, AV_LOG_VERBOSE, "WHIP: Got state=%d, answer: %s\n", whip->state, whip->sdp_answer); + av_log(whip, AV_LOG_VERBOSE, "Got state=%d, answer: %s\n", whip->state, whip->sdp_answer); end: ffurl_closep(&whip_uc); @@ -851,7 +891,7 @@ static int parse_answer(AVFormatContext *s) WHIPContext *whip = s->priv_data; if (!whip->sdp_answer || !strlen(whip->sdp_answer)) { - av_log(whip, AV_LOG_ERROR, "WHIP: No answer to parse\n"); + av_log(whip, AV_LOG_ERROR, "No answer to parse\n"); ret = AVERROR(EINVAL); goto end; } @@ -879,16 +919,24 @@ static int parse_answer(AVFormatContext *s) if (ptr && av_stristr(ptr, "host")) { char protocol[17], host[129]; int priority, port; +#if HAVE_STRUCT_SOCKADDR_IN6 + struct in6_addr addr6; +#endif ret = sscanf(ptr, "%16s %d %128s %d typ host", protocol, &priority, host, &port); if (ret != 4) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed %d to parse line %d %s from %s\n", + av_log(whip, AV_LOG_ERROR, "Failed %d to parse line %d %s from %s\n", ret, i, line, whip->sdp_answer); ret = AVERROR(EIO); goto end; } - +#if HAVE_STRUCT_SOCKADDR_IN6 + if (whip->flags & WHIP_FLAG_IGNORE_IPV6 && inet_pton(AF_INET6, host, &addr6) == 1) { + av_log(whip, AV_LOG_DEBUG, "Ignoring IPv6 ICE candidates %s, line %d %s \n", host, i, line); + continue; + } +#endif if (av_strcasecmp(protocol, "udp")) { - av_log(whip, AV_LOG_ERROR, "WHIP: Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n", + av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n", protocol, i, line, whip->sdp_answer); ret = AVERROR(EIO); goto end; @@ -906,19 +954,19 @@ static int parse_answer(AVFormatContext *s) } if (!whip->ice_pwd_remote || !strlen(whip->ice_pwd_remote)) { - av_log(whip, AV_LOG_ERROR, "WHIP: No remote ice pwd parsed from %s\n", whip->sdp_answer); + av_log(whip, AV_LOG_ERROR, "No remote ice pwd parsed from %s\n", whip->sdp_answer); ret = AVERROR(EINVAL); goto end; } if (!whip->ice_ufrag_remote || !strlen(whip->ice_ufrag_remote)) { - av_log(whip, AV_LOG_ERROR, "WHIP: No remote ice ufrag parsed from %s\n", whip->sdp_answer); + av_log(whip, AV_LOG_ERROR, "No remote ice ufrag parsed from %s\n", whip->sdp_answer); ret = AVERROR(EINVAL); goto end; } if (!whip->ice_protocol || !whip->ice_host || !whip->ice_port) { - av_log(whip, AV_LOG_ERROR, "WHIP: No ice candidate parsed from %s\n", whip->sdp_answer); + av_log(whip, AV_LOG_ERROR, "No ice candidate parsed from %s\n", whip->sdp_answer); ret = AVERROR(EINVAL); goto end; } @@ -926,7 +974,7 @@ static int parse_answer(AVFormatContext *s) if (whip->state < WHIP_STATE_NEGOTIATED) whip->state = WHIP_STATE_NEGOTIATED; whip->whip_answer_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: SDP state=%d, offer=%luB, answer=%luB, ufrag=%s, pwd=%luB, transport=%s://%s:%d, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "SDP state=%d, offer=%luB, answer=%luB, ufrag=%s, pwd=%luB, transport=%s://%s:%d, elapsed=%dms\n", whip->state, strlen(whip->sdp_offer), strlen(whip->sdp_answer), whip->ice_ufrag_remote, strlen(whip->ice_pwd_remote), whip->ice_protocol, whip->ice_host, whip->ice_port, ELAPSED(whip->whip_starttime, av_gettime())); @@ -977,7 +1025,7 @@ static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, in /* The username is the concatenation of the two ICE ufrag */ ret = snprintf(username, sizeof(username), "%s:%s", whip->ice_ufrag_remote, whip->ice_ufrag_local); if (ret <= 0 || ret >= sizeof(username)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to build username %s:%s, max=%lu, ret=%d\n", + av_log(whip, AV_LOG_ERROR, "Failed to build username %s:%s, max=%lu, ret=%d\n", whip->ice_ufrag_remote, whip->ice_ufrag_local, sizeof(username), ret); ret = AVERROR(EIO); goto end; @@ -1046,7 +1094,7 @@ static int ice_create_response(AVFormatContext *s, char *tid, int tid_size, uint WHIPContext *whip = s->priv_data; if (tid_size != 12) { - av_log(whip, AV_LOG_ERROR, "WHIP: Invalid transaction ID size. Expected 12, got %d\n", tid_size); + av_log(whip, AV_LOG_ERROR, "Invalid transaction ID size. Expected 12, got %d\n", tid_size); return AVERROR(EINVAL); } @@ -1149,7 +1197,7 @@ static int ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_siz return ret; if (buf_size < ICE_STUN_HEADER_SIZE) { - av_log(whip, AV_LOG_ERROR, "WHIP: Invalid STUN message, expected at least %d, got %d\n", + av_log(whip, AV_LOG_ERROR, "Invalid STUN message, expected at least %d, got %d\n", ICE_STUN_HEADER_SIZE, buf_size); return AVERROR(EINVAL); } @@ -1160,13 +1208,13 @@ static int ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_siz /* Build the STUN binding response. */ ret = ice_create_response(s, tid, sizeof(tid), whip->buf, sizeof(whip->buf), &size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to create STUN binding response, size=%d\n", size); + av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding response, size=%d\n", size); return ret; } ret = ffurl_write(whip->udp, whip->buf, size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to send STUN binding response, size=%d\n", size); + av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding response, size=%d\n", size); return ret; } @@ -1196,7 +1244,7 @@ static int udp_connect(AVFormatContext *s) ret = ffurl_open_whitelist(&whip->udp, url, AVIO_FLAG_WRITE, &s->interrupt_callback, &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to connect udp://%s:%d\n", whip->ice_host, whip->ice_port); + av_log(whip, AV_LOG_ERROR, "Failed to connect udp://%s:%d\n", whip->ice_host, whip->ice_port); goto end; } @@ -1207,7 +1255,7 @@ static int udp_connect(AVFormatContext *s) if (whip->state < WHIP_STATE_UDP_CONNECTED) whip->state = WHIP_STATE_UDP_CONNECTED; whip->whip_udp_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: UDP state=%d, elapsed=%dms, connected to udp://%s:%d\n", + av_log(whip, AV_LOG_VERBOSE, "UDP state=%d, elapsed=%dms, connected to udp://%s:%d\n", whip->state, ELAPSED(whip->whip_starttime, av_gettime()), whip->ice_host, whip->ice_port); end: @@ -1225,7 +1273,7 @@ static int ice_dtls_handshake(AVFormatContext *s) char buf[256], *cert_buf = NULL, *key_buf = NULL; if (whip->state < WHIP_STATE_UDP_CONNECTED || !whip->udp) { - av_log(whip, AV_LOG_ERROR, "WHIP: UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp); + av_log(whip, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp); return AVERROR(EINVAL); } @@ -1234,13 +1282,13 @@ static int ice_dtls_handshake(AVFormatContext *s) /* Build the STUN binding request. */ ret = ice_create_request(s, whip->buf, sizeof(whip->buf), &size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to create STUN binding request, size=%d\n", size); + av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size); goto end; } ret = ffurl_write(whip->udp, whip->buf, size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to send STUN binding request, size=%d\n", size); + av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size); goto end; } @@ -1255,7 +1303,7 @@ static int ice_dtls_handshake(AVFormatContext *s) now = av_gettime(); if (now - starttime >= whip->handshake_timeout * 1000) { - av_log(whip, AV_LOG_ERROR, "WHIP: DTLS handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n", + av_log(whip, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n", whip->handshake_timeout, ELAPSED(starttime, now), ELAPSED(whip->whip_starttime, now), whip->state); ret = AVERROR(ETIMEDOUT); goto end; @@ -1270,7 +1318,7 @@ static int ice_dtls_handshake(AVFormatContext *s) av_usleep(5 * 1000); continue; } - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read message\n"); + av_log(whip, AV_LOG_ERROR, "Failed to read message\n"); goto end; } @@ -1283,7 +1331,7 @@ static int ice_dtls_handshake(AVFormatContext *s) if (whip->state < WHIP_STATE_ICE_CONNECTED) { whip->state = WHIP_STATE_ICE_CONNECTED; whip->whip_ice_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n", whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "", whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, av_gettime())); @@ -1383,20 +1431,26 @@ static int setup_srtp(AVFormatContext *s) /* Setup SRTP context for outgoing packets */ if (!av_base64_encode(buf, sizeof(buf), send_key, sizeof(send_key))) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to encode send key\n"); + av_log(whip, AV_LOG_ERROR, "Failed to encode send key\n"); ret = AVERROR(EIO); goto end; } ret = ff_srtp_set_crypto(&whip->srtp_audio_send, suite, buf); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to set crypto for audio send\n"); + av_log(whip, AV_LOG_ERROR, "Failed to set crypto for audio send\n"); goto end; } ret = ff_srtp_set_crypto(&whip->srtp_video_send, suite, buf); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to set crypto for video send\n"); + av_log(whip, AV_LOG_ERROR, "Failed to set crypto for video send\n"); + goto end; + } + + ret = ff_srtp_set_crypto(&whip->srtp_video_rtx_send, suite, buf); + if (ret < 0) { + av_log(whip, AV_LOG_ERROR, "WHIP: Failed to set crypto for video rtx send\n"); goto end; } @@ -1408,27 +1462,58 @@ static int setup_srtp(AVFormatContext *s) /* Setup SRTP context for incoming packets */ if (!av_base64_encode(buf, sizeof(buf), recv_key, sizeof(recv_key))) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to encode recv key\n"); + av_log(whip, AV_LOG_ERROR, "Failed to encode recv key\n"); ret = AVERROR(EIO); goto end; } ret = ff_srtp_set_crypto(&whip->srtp_recv, suite, buf); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to set crypto for recv\n"); + av_log(whip, AV_LOG_ERROR, "Failed to set crypto for recv\n"); goto end; } if (whip->state < WHIP_STATE_SRTP_FINISHED) whip->state = WHIP_STATE_SRTP_FINISHED; whip->whip_srtp_time = av_gettime(); - av_log(whip, AV_LOG_VERBOSE, "WHIP: SRTP setup done, state=%d, suite=%s, key=%luB, elapsed=%dms\n", + av_log(whip, AV_LOG_VERBOSE, "SRTP setup done, state=%d, suite=%s, key=%luB, elapsed=%dms\n", whip->state, suite, sizeof(send_key), ELAPSED(whip->whip_starttime, av_gettime())); end: return ret; } + +/** + * RTX history helpers + */ + static void rtp_history_store(WHIPContext *whip, const uint8_t *buf, int size) +{ + int pos = whip->hist_head % whip->history_size; + RtpHistoryItem *it = &whip->history[pos]; + /* free older entry */ + av_free(it->buf); + it->buf = av_malloc(size); + if (!it->buf) + return; + + memcpy(it->buf, buf, size); + it->size = size; + it->seq = AV_RB16(buf + 2); + + whip->hist_head = ++pos; +} + +static const RtpHistoryItem *rtp_history_find(const WHIPContext *whip, uint16_t seq) +{ + for (int i = 0; i < whip->history_size; i++) { + const RtpHistoryItem *it = &whip->history[i]; + if (it->buf && it->seq == seq) + return it; + } + return NULL; +} + /** * Callback triggered by the RTP muxer when it creates and sends out an RTP packet. * @@ -1461,18 +1546,61 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size) /* Encrypt by SRTP and send out. */ cipher_size = ff_srtp_encrypt(srtp, buf, buf_size, whip->buf, sizeof(whip->buf)); if (cipher_size <= 0 || cipher_size < buf_size) { - av_log(whip, AV_LOG_WARNING, "WHIP: Failed to encrypt packet=%dB, cipher=%dB\n", buf_size, cipher_size); + av_log(whip, AV_LOG_WARNING, "Failed to encrypt packet=%dB, cipher=%dB\n", buf_size, cipher_size); return 0; } + /* Store only ORIGINAL video packets (non-RTX, non-RTCP) */ + if (!is_rtcp && is_video) + rtp_history_store(whip, buf, buf_size); + ret = ffurl_write(whip->udp, whip->buf, cipher_size); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write packet=%dB, ret=%d\n", cipher_size, ret); + av_log(whip, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret); return ret; } return ret; } +/** + * See https://datatracker.ietf.org/doc/html/rfc4588 + * Build and send a single RTX packet + */ +static int send_rtx_packet(AVFormatContext *s, const uint8_t *orig_pkt_buf, int orig_size) +{ + WHIPContext *whip = s->priv_data; + int new_size, cipher_size; + if (whip->flags & WHIP_FLAG_DISABLE_RTX) + return 0; + + /* allocate new buffer: header + 2 + payload */ + if (orig_size + 2 > sizeof(whip->buf)) + return 0; + + memcpy(whip->buf, orig_pkt_buf, orig_size); + + uint8_t *hdr = whip->buf; + uint16_t orig_seq = AV_RB16(hdr + 2); + + /* rewrite header */ + hdr[1] = (hdr[1] & 0x80) | whip->rtx_payload_type; /* keep M bit */ + AV_WB16(hdr + 2, whip->rtx_seq++); + AV_WB32(hdr + 8, whip->video_rtx_ssrc); + + /* shift payload 2 bytes */ + memmove(hdr + 12 + 2, hdr + 12, orig_size - 12); + AV_WB16(hdr + 12, orig_seq); + + new_size = orig_size + 2; + + /* Encrypt by SRTP and send out. */ + cipher_size = ff_srtp_encrypt(&whip->srtp_video_rtx_send, whip->buf, new_size, whip->buf, sizeof(whip->buf)); + if (cipher_size <= 0 || cipher_size < new_size) { + av_log(whip, AV_LOG_WARNING, "WHIP: Failed to encrypt packet=%dB, cipher=%dB\n", new_size, cipher_size); + return 0; + } + return ffurl_write(whip->udp, whip->buf, cipher_size); +} /** * Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP @@ -1496,7 +1624,7 @@ static int create_rtp_muxer(AVFormatContext *s) const AVOutputFormat *rtp_format = av_guess_format("rtp", NULL, NULL); if (!rtp_format) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to guess rtp muxer\n"); + av_log(whip, AV_LOG_ERROR, "Failed to guess rtp muxer\n"); ret = AVERROR(ENOSYS); goto end; } @@ -1564,7 +1692,7 @@ static int create_rtp_muxer(AVFormatContext *s) ret = avformat_write_header(rtp_ctx, &opts); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write rtp header\n"); + av_log(whip, AV_LOG_ERROR, "Failed to write rtp header\n"); goto end; } @@ -1576,7 +1704,7 @@ static int create_rtp_muxer(AVFormatContext *s) if (whip->state < WHIP_STATE_READY) whip->state = WHIP_STATE_READY; - av_log(whip, AV_LOG_INFO, "WHIP: Muxer state=%d, buffer_size=%d, max_packet_size=%d, " + av_log(whip, AV_LOG_INFO, "Muxer state=%d, buffer_size=%d, max_packet_size=%d, " "elapsed=%dms(init:%d,offer:%d,answer:%d,udp:%d,ice:%d,dtls:%d,srtp:%d)\n", whip->state, buffer_size, max_packet_size, ELAPSED(whip->whip_starttime, av_gettime()), ELAPSED(whip->whip_starttime, whip->whip_init_time), @@ -1618,7 +1746,7 @@ static int dispose_session(AVFormatContext *s) if (whip->authorization) ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization); if (ret <= 0 || ret >= sizeof(buf)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to generate headers, size=%d, %s\n", ret, buf); + av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf); ret = AVERROR(EINVAL); goto end; } @@ -1629,7 +1757,7 @@ static int dispose_session(AVFormatContext *s) ret = ffurl_open_whitelist(&whip_uc, whip->whip_resource_url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback, &opts, s->protocol_whitelist, s->protocol_blacklist, NULL); if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to DELETE url=%s\n", whip->whip_resource_url); + av_log(whip, AV_LOG_ERROR, "Failed to DELETE url=%s\n", whip->whip_resource_url); goto end; } @@ -1640,12 +1768,12 @@ static int dispose_session(AVFormatContext *s) break; } if (ret < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read response from DELETE url=%s\n", whip->whip_resource_url); + av_log(whip, AV_LOG_ERROR, "Failed to read response from DELETE url=%s\n", whip->whip_resource_url); goto end; } } - av_log(whip, AV_LOG_INFO, "WHIP: Dispose resource %s ok\n", whip->whip_resource_url); + av_log(whip, AV_LOG_INFO, "Dispose resource %s ok\n", whip->whip_resource_url); end: ffurl_closep(&whip_uc); @@ -1791,18 +1919,78 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) if (ret > 0) { if (is_dtls_packet(whip->buf, ret)) { if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to handle DTLS message\n"); + av_log(whip, AV_LOG_ERROR, "Failed to handle DTLS message\n"); goto end; } } + /** + * Handle RTCP NACK + * Refer to RFC 4585, Section 6.2.1 + * The Generic NACK message is identified by PT=RTPFB and FMT=1. + * TODO: disable retransmisstion when "-tune zerolatency" + */ + if (media_is_rtcp(whip->buf, ret)) { + int ptr = 0; + uint8_t pt = whip->buf[ptr + 1]; + uint8_t fmt = (whip->buf[ptr] & 0x1f); + if (ptr + 4 <= ret && pt == 205 && fmt == 1) { + /** + * Refer to RFC 3550, Section 6.4.1. + * The length of this RTCP packet in 32-bit words minus one, + * including the header and any padding. + */ + int rtcp_len = (AV_RB16(&whip->buf[ptr + 2]) + 1) * 4; + /* SRTCP index(4 bytes) + HMAC (SRTP_AES128_CM_SHA1_80 10bytes) */ + int srtcp_len = rtcp_len + 4 + 10; + if (srtcp_len == ret && rtcp_len >= 12) { + int i = 0; + uint8_t *pkt = av_malloc(srtcp_len); + memcpy(pkt, whip->buf, srtcp_len); + int ret = ff_srtp_decrypt(&whip->srtp_recv, pkt, &srtcp_len); + if (ret < 0) + av_log(whip, AV_LOG_ERROR, "WHIP: SRTCP decrypt failed: %d\n", ret); + while (12 + i < rtcp_len && ret == 0) { + /** + * See https://datatracker.ietf.org/doc/html/rfc4585#section-6.1 + * Handle multi NACKs in bundled packet. + */ + uint16_t pid = AV_RB16(&pkt[ptr + 12 + i]); + uint16_t blp = AV_RB16(&pkt[ptr + 14 + i]); + + /* retransmit pid + any bit set in blp */ + for (int bit = -1; bit < 16; bit++) { + uint16_t seq = (bit < 0) ? pid : pid + bit + 1; + if (bit >= 0 && !(blp & (1 << bit))) + continue; + + const RtpHistoryItem *it = rtp_history_find(whip, seq); + if (it) { + av_log(whip, AV_LOG_VERBOSE, + "WHIP: NACK, packet found: size: %d, seq=%d, rtx size=%d, lateset stored packet seq:%d\n", + it->size, seq, ret, whip->history[whip->hist_head-1].seq); + ret = send_rtx_packet(s, it->buf, it->size); + if (ret <= 0 && !(whip->flags & WHIP_FLAG_DISABLE_RTX)) + av_log(whip, AV_LOG_ERROR, "WHIP: Failed to send RTX packet\n"); + } else { + av_log(whip, AV_LOG_VERBOSE, + "WHIP: NACK, packet not found, seq=%d, latest stored packet seq: %d, latest rtx seq: %d\n", + seq, whip->history[whip->hist_head-1].seq, whip->rtx_seq); + } + } + i = i + 4; + } + av_free(pkt); + } + } + } } else if (ret != AVERROR(EAGAIN)) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read from UDP socket\n"); + av_log(whip, AV_LOG_ERROR, "Failed to read from UDP socket\n"); goto end; } if (whip->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) { if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) { - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to insert SPS/PPS before IDR\n"); + av_log(whip, AV_LOG_ERROR, "Failed to insert SPS/PPS before IDR\n"); goto end; } } @@ -1810,10 +1998,10 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) ret = ff_write_chained(rtp_ctx, 0, pkt, s, 0); if (ret < 0) { if (ret == AVERROR(EINVAL)) { - av_log(whip, AV_LOG_WARNING, "WHIP: Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret); + av_log(whip, AV_LOG_WARNING, "Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret); ret = 0; } else - av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write packet, size=%d\n", pkt->size); + av_log(whip, AV_LOG_ERROR, "Failed to write packet, size=%d\n", pkt->size); goto end; } @@ -1834,7 +2022,7 @@ static av_cold void whip_deinit(AVFormatContext *s) ret = dispose_session(s); if (ret < 0) - av_log(whip, AV_LOG_WARNING, "WHIP: Failed to dispose resource, ret=%d\n", ret); + av_log(whip, AV_LOG_WARNING, "Failed to dispose resource, ret=%d\n", ret); for (i = 0; i < s->nb_streams; i++) { AVFormatContext* rtp_ctx = s->streams[i]->priv_data; @@ -1881,7 +2069,7 @@ static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket extradata_isom = st->codecpar->extradata_size > 0 && st->codecpar->extradata[0] == 1; if (pkt->size >= 5 && AV_RB32(b) != 0x0000001 && (AV_RB24(b) != 0x000001 || extradata_isom)) { ret = ff_stream_add_bitstream_filter(st, "h264_mp4toannexb", NULL); - av_log(whip, AV_LOG_VERBOSE, "WHIP: Enable BSF h264_mp4toannexb, packet=[%x %x %x %x %x ...], extradata_isom=%d\n", + av_log(whip, AV_LOG_VERBOSE, "Enable BSF h264_mp4toannexb, packet=[%x %x %x %x %x ...], extradata_isom=%d\n", b[0], b[1], b[2], b[3], b[4], extradata_isom); } else whip->h264_annexb_insert_sps_pps = 1; @@ -1893,11 +2081,15 @@ static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket #define OFFSET(x) offsetof(WHIPContext, x) #define ENC AV_OPT_FLAG_ENCODING_PARAM static const AVOption options[] = { - { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC }, - { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC }, - { "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, - { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, - { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC }, + { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC }, + { "authorization", "Optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "cert_file", "Optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "key_file", "Optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "whip_flags", "Set flags affecting WHIP connection behavior", OFFSET(flags), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, 0, ENC, .unit = "flags" }, + { "ignore_ipv6", "Ignore any IPv6 ICE candidate", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_IGNORE_IPV6 }, 0, UINT_MAX, ENC, .unit = "flags" }, + { "disable_rtx", "Disable RFC 4588 RTX", 0, AV_OPT_TYPE_CONST, { .i64 = WHIP_FLAG_DISABLE_RTX }, 0, UINT_MAX, ENC, .unit = "flags" }, + { "rtx_history_size", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, 2048, ENC }, { NULL }, };