マルチコーデックサイマルキャストの送信に対応する

.github/workflows/build.yml

@@ -32,8 +32,11 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           sudo apt-get update
-          sudo apt-get install -y libpulse-dev
+          sudo apt-get install -y libpulse-dev yasm clang-12
         if: matrix.m.linux == true
+      - run: |
+          brew install yasm
+        if: matrix.m.linux == false
       - run: python3 run.py --sumomo --package ${{ matrix.m.name }}
       - name: Get package name
         run: |

.vscode/c_cpp_properties.json

@@ -5,14 +5,20 @@
             "includePath": [
                 "${workspaceFolder}/include",
                 "${workspaceFolder}/_source/libdatachannel/include",
+                "${workspaceFolder}/_source/libdatachannel/include/rtc",
+                "${workspaceFolder}/_source/libdatachannel/src",
                 "${workspaceFolder}/_source/libdatachannel/deps/json/include",
                 "${workspaceFolder}/_source/libdatachannel/deps/plog/include",
+                "${workspaceFolder}/_source/libdatachannel/deps/libjuice/include",
+                "${workspaceFolder}/_source/libdatachannel/deps/usrsctp/usrsctplib",
+                "${workspaceFolder}/_source/libdatachannel/deps/libsrtp/include",
                 "${workspaceFolder}/_source/opus/include",
 
                 "${workspaceFolder}/_build/ubuntu-20.04_x86_64/release/sorac",
                 "${workspaceFolder}/_build/ubuntu-20.04_x86_64/release/sorac/proto/sorac",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/mbedtls/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/openh264/include",
+                "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/aom/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/libjpeg-turbo/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/libyuv/include",
 
@@ -27,6 +33,7 @@
                 "${workspaceFolder}/_build/macos_arm64/release/sorac/proto/sorac",
                 "${workspaceFolder}/_install/macos_arm64/release/mbedtls/include",
                 "${workspaceFolder}/_install/macos_arm64/release/openh264/include",
+                "${workspaceFolder}/_install/macos_arm64/release/aom/include",
                 "${workspaceFolder}/_install/macos_arm64/release/libjpeg-turbo/include",
                 "${workspaceFolder}/_install/macos_arm64/release/libyuv/include"
             ],

.vscode/launch.json

@@ -0,0 +1,41 @@
+{
+    // IntelliSense を使用して利用可能な属性を学べます。
+    // 既存の属性の説明をホバーして表示します。
+    // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "(gdb) 起動",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/_build/ubuntu-24.04_x86_64/debug/examples/sumomo/sumomo",
+            "args": [
+                "--signaling-url=wss://sora.wandbox.org/signaling",
+                "--channel-id=sora",
+                "--capture-type=fake-i420",
+                "--video-codec-type=AV1",
+                "--audio-type=fake",
+                "--aom=${workspaceFolder}/_install/ubuntu-24.04_x86_64/debug/aom/lib/libaom.so",
+                "--openh264=${workspaceFolder}/libopenh264-2.6.0-linux64.8.so",
+            ],
+            "stopAtEntry": false,
+            "cwd": "${fileDirname}",
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "gdb の再フォーマットを有効にする",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "逆アセンブリ フレーバーを Intel に設定",
+                    "text": "-gdb-set disassembly-flavor intel",
+                    "ignoreFailures": true
+                }
+            ]
+        }
+
+    ]
+}

.vscode/settings.json

@@ -103,6 +103,8 @@
         "recorder.h": "c",
         "__mutex_base": "cpp",
         "sorac.h": "c",
-        "soracp.json.c.h": "c"
+        "soracp.json.c.h": "c",
+        "complex": "cpp",
+        "cfenv": "cpp"
     }
 }

CHANGES.md

@@ -11,6 +11,12 @@
 
 ## develop
 
+- [ADD] マルチコーデックサイマルキャストの送信に対応する
+  - @melpon
+- [ADD] libaom を追加して AV1 エンコードできるようにする
+  - @melpon
+- [UPDATE] protoc-gen-jsonif をアップデートして optional 対応する
+  - @melpon
 - [ADD] Ubuntu 24.04 x86_64 対応
   - @melpon
 

CMakeLists.txt

@@ -4,6 +4,7 @@ cmake_policy(SET CMP0074 NEW)
 
 project(sorac C CXX)
 
+set(BUILD_SHARED_LIBS OFF)
 add_subdirectory(_source/libdatachannel libdatachannel EXCLUDE_FROM_ALL)
 add_subdirectory(_source/libdatachannel/deps/json json EXCLUDE_FROM_ALL)
 add_subdirectory(_source/opus opus EXCLUDE_FROM_ALL)
@@ -92,8 +93,11 @@ add_custom_command(
 target_sources(sorac
   PRIVATE
     "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.cpp"
+    src/aom_av1_video_encoder.cpp
     src/current_time.cpp
     src/data_channel.cpp
+    src/default_encoder_adapter.cpp
+    src/h264_profile_level_id.cpp
     src/open_h264_video_encoder.cpp
     src/opus_audio_encoder.cpp
     src/signaling.cpp
@@ -112,9 +116,12 @@ target_sources(sorac
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.h"
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.h"
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.hpp"
+      include/sorac/aom_av1_video_encoder.hpp
       include/sorac/bitrate.hpp
       include/sorac/current_time.hpp
       include/sorac/data_channel.hpp
+      include/sorac/default_encoder_adapter.hpp
+      include/sorac/h264_profile_level_id.hpp
       include/sorac/open_h264_video_encoder.hpp
       include/sorac/opus_audio_encoder.hpp
       include/sorac/signaling.hpp
@@ -132,13 +139,17 @@ target_compile_definitions(sorac PRIVATE JSONIF_USE_NLOHMANN_JSON)
 # OpenH264
 target_include_directories(sorac PRIVATE ${OPENH264_ROOT_DIR}/include)
 
+# AOM
+target_include_directories(sorac PRIVATE ${AOM_ROOT_DIR}/include)
+
 set_target_properties(sorac PROPERTIES CXX_STANDARD 20 C_STANDARD 20)
 
 set(_LIBS
   $<TARGET_FILE:LibDataChannel::LibDataChannelStatic>
   ${MbedTLS_LIBRARY}
   ${MbedCrypto_LIBRARY}
   ${MbedX509_LIBRARY}
+  ${AOM_ROOT_DIR}/lib/libaom.a
   $<TARGET_FILE:Opus::opus>
   $<TARGET_FILE:srtp2>
   $<TARGET_FILE:LibJuice::LibJuiceStatic>

NOTICE.md

@@ -333,3 +333,68 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ```
+
+## AOM
+
+https://aomedia.googlesource.com/aom/
+
+```
+Copyright (c) 2016, Alliance for Open Media. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in
+   the documentation and/or other materials provided with the
+   distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+```
+
+## WebRTC
+
+https://webrtc.googlesource.com/src/
+
+```
+Copyright (c) 2011, The WebRTC project authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+  * Neither the name of Google nor the names of its contributors may
+    be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```

VERSION

@@ -1,8 +1,10 @@
 SORA_C_SDK_VERSION=2024.1.0
-LIBDATACHANNEL_VERSION=v0.21.1
+LIBDATACHANNEL_URL=https://github.com/melpon/libdatachannel.git
+LIBDATACHANNEL_VERSION=f32b20132a953a2dff72ecdd106a6771fa0f8e30
 OPUS_VERSION=v1.4
-CMAKE_VERSION=3.28.1
-OPENH264_VERSION=v2.4.0
+CMAKE_VERSION=3.31.6
+OPENH264_VERSION=v2.6.0
+AOM_VERSION=v3.12.0
 MBEDTLS_VERSION=v3.5.1
 PROTOBUF_VERSION=21.1
 PROTOC_GEN_JSONIF_VERSION=0.12.1

doc/FAQ.md

@@ -80,6 +80,26 @@ libwebrtc ベースの [Sora C++ SDK](https://github.com/shiguredo/sora-cpp-sdk)
     --cacert cacert.pem
 ```
 
+### マルチコーデックサイマルキャスト
+
+マルチコーデックサイマルキャストを利用した配信を macOS arm64 で利用する例です。
+
+```bash
+./sumomo --signaling-url wss://sora.example.com/signaling \
+  --channel-id sora \
+  --simulcast true \
+  --simulcast-multicodec true \
+  --audio=true \
+  --audio-type=macos \
+  --capture-type mac \
+  --capture-device-name=OBS \
+  --video-codec-type=AV1 \
+  --video-bit-rate 5000 \
+  --h264-encoder-type videotoolbox \
+  --aom `pwd`/_install/macos_arm64/release/aom/lib/libaom.dylib
+~/shiguredo/sora-c-sdk
+```
+
 ## Sumomo のヘルプ
 
 ```bash

examples/sumomo/fake_capturer.cpp

@@ -13,7 +13,8 @@ namespace sumomo {
 
 class FakeCapturer : public SumomoCapturer {
  public:
-  FakeCapturer() {
+  FakeCapturer(int width, int height, int fps, SumomoFakeCapturerFormat format)
+      : width_(width), height_(height), fps_(fps), format_(format) {
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::FakeCapturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -42,26 +43,39 @@ class FakeCapturer : public SumomoCapturer {
       engine_ = std::make_unique<std::mt19937>(seed_gen());
       return nullptr;
     });
-    th_.Start(30, [this](std::chrono::microseconds timestamp,
-                         std::chrono::microseconds prev) {
-      std::uniform_int_distribution<int> dist(0, 640 * 480 - 1);
+    th_.Start(fps_, [this](std::chrono::microseconds timestamp,
+                           std::chrono::microseconds prev) {
+      std::uniform_int_distribution<int> dist(0, width_ * height_ - 1);
       sorac::VideoFrame frame;
       frame.timestamp = timestamp;
-      frame.i420_buffer = sorac::VideoFrameBufferI420::Create(640, 480);
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.base_width = 640;
-      frame.base_height = 480;
+      frame.frame_number = ++frame_number_;
+      if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_I420) {
+        frame.i420_buffer =
+            sorac::VideoFrameBufferI420::Create(width_, height_);
+        for (int i = 0; i < width_ / 10; i++) {
+          frame.i420_buffer->y[dist(*engine_)] = 0xff;
+        }
+      } else if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_NV12) {
+        frame.nv12_buffer =
+            sorac::VideoFrameBufferNV12::Create(width_, height_);
+        for (int i = 0; i < width_ / 10; i++) {
+          frame.nv12_buffer->y[dist(*engine_)] = 0xff;
+        }
+      }
+      frame.base_width = width_;
+      frame.base_height = height_;
       callback_(frame);
     });
     return 0;
   }
   void Stop() { th_.Stop(); }
 
  private:
+  int width_;
+  int height_;
+  int fps_;
+  int frame_number_ = 0;
+  SumomoFakeCapturerFormat format_;
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   SteadyFrameThread th_;
   std::unique_ptr<std::mt19937> engine_;
@@ -71,7 +85,10 @@ class FakeCapturer : public SumomoCapturer {
 
 extern "C" {
 
-SumomoCapturer* sumomo_fake_capturer_create() {
-  return new sumomo::FakeCapturer();
+SumomoCapturer* sumomo_fake_capturer_create(int width,
+                                            int height,
+                                            int fps,
+                                            SumomoFakeCapturerFormat format) {
+  return new sumomo::FakeCapturer(width, height, fps, format);
 }
 }

examples/sumomo/fake_capturer.h

@@ -9,7 +9,16 @@
 extern "C" {
 #endif
 
-extern SumomoCapturer* sumomo_fake_capturer_create();
+typedef enum {
+  SUMOMO_FAKE_CAPTURER_FORMAT_I420 = 0,
+  SUMOMO_FAKE_CAPTURER_FORMAT_NV12 = 1,
+} SumomoFakeCapturerFormat;
+
+extern SumomoCapturer* sumomo_fake_capturer_create(
+    int width,
+    int height,
+    int fps,
+    SumomoFakeCapturerFormat format);
 
 #ifdef __cplusplus
 }

examples/sumomo/mac_capturer.h

@@ -11,7 +11,8 @@ extern "C" {
 
 extern SumomoCapturer* sumomo_mac_capturer_create(const char* device,
                                                   int width,
-                                                  int height);
+                                                  int height,
+                                                  int fps);
 
 #ifdef __cplusplus
 }