Merge branch 'main' into audio-decoding-module

Author: vltkv

apps/common-app/src/examples/Worklets/Worklets.tsx

@@ -3,9 +3,9 @@ import { Text, View, StyleSheet } from 'react-native';
 import {
   AudioContext,
   AudioManager,
-  AudioRecorder,
-  RecorderAdapterNode,
-  WorkletNode
+  WorkletNode,
+  WorkletProcessingNode,
+  WorkletSourceNode
 } from 'react-native-audio-api';
 import { Container, Button } from "../../components";
 import { Extrapolation, useSharedValue } from "react-native-reanimated";
@@ -16,13 +16,12 @@ import Animated, {
 } from "react-native-reanimated";
 import { colors } from "../../styles";
 
-
 function Worklets() {
   const SAMPLE_RATE = 44100;
-  const recorderRef = useRef<AudioRecorder | null>(null);
   const aCtxRef = useRef<AudioContext | null>(null);
-  const recorderAdapterRef = useRef<RecorderAdapterNode | null>(null);
   const workletNodeRef = useRef<WorkletNode | null>(null);
+  const workletProcessingNodeRef = useRef<WorkletProcessingNode | null>(null);
+  const workletSourceNodeRef = useRef<WorkletSourceNode | null>(null);
 
   const bar0 = useSharedValue(0);
   const bar1 = useSharedValue(0);
@@ -36,65 +35,71 @@ function Worklets() {
       iosMode: 'spokenAudio',
       iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'],
     });
-
-    AudioManager.requestRecordingPermissions();
-    recorderRef.current = new AudioRecorder({
-      sampleRate: SAMPLE_RATE,
-      bufferLengthInSamples: 1024,
-    });
   }, []);
 
   const start = () => {
-    if (!recorderRef.current) {
-      console.error("Recorder is not initialized");
-      return;
+
+    const processingWorklet = (inputAudioData: Array<Float32Array>, outputAudioData: Array<Float32Array>, framesToProcess: number, currentTime: number) => {
+      'worklet';
+      const gain = 0.5;
+      for (let channel = 0; channel < inputAudioData.length; channel++) {
+        const inputChannelData = inputAudioData[channel];
+        const outputChannelData = outputAudioData[channel];
+        for (let i = 0; i < framesToProcess; i++) {
+          outputChannelData[i] = inputChannelData[i] * gain;
+        }
+      }
+    };
+
+    const sourceWorklet = (audioData: Array<Float32Array>, framesToProcess: number, currentTime: number, startOffset: number) => {
+      'worklet';
+      for (let i = 0; i < audioData.length; i++) {
+        for (let j = 0; j < framesToProcess; j++) {
+          audioData[i][j] = Math.sin((currentTime + startOffset + j));
+        }
+      }
     }
 
     const worklet = (audioData: Array<Float32Array>, inputChannelCount: number) => {
       'worklet';
 
-
       // Calculates RMS amplitude
       let sum = 0;
       for (let i = 0; i < audioData.length; i++) {
         sum += audioData[0][i] * audioData[0][i];
       }
       const rms = Math.sqrt(sum / audioData[0].length);
-      const scaledAmplitude = Math.min(rms * 1000, 1);
+      const scaledAmplitude = Math.min(rms * 10, 1);
 
-      console.log(`RMS: ${rms}, Scaled: ${scaledAmplitude}`);
+      // console.log(`RMS: ${rms}, Scaled: ${scaledAmplitude}`);
 
       bar0.value = withSpring(bar1.value, { damping: 20, stiffness: 150 });
       bar1.value = withSpring(bar2.value, { damping: 20, stiffness: 150 });
       bar3.value = withSpring(bar2.value, { damping: 20, stiffness: 150 });
       bar4.value = withSpring(bar3.value, { damping: 20, stiffness: 150 });
-      bar2.value = withSpring(scaledAmplitude, { damping: 20, stiffness: 200 });
+      bar2.value = withSpring(scaledAmplitude, { damping: 20, stiffness: 150 });
     };
 
     aCtxRef.current = new AudioContext({ sampleRate: SAMPLE_RATE });
-    recorderAdapterRef.current = aCtxRef.current.createRecorderAdapter();
-    workletNodeRef.current = aCtxRef.current.createWorkletNode(worklet, 512, 1);
-    recorderAdapterRef.current.connect(workletNodeRef.current);
+    workletSourceNodeRef.current = aCtxRef.current.createWorkletSourceNode(sourceWorklet, 'AudioRuntime');
+    workletNodeRef.current = aCtxRef.current.createWorkletNode(worklet, 256, 1, 'UIRuntime');
+    workletProcessingNodeRef.current = aCtxRef.current.createWorkletProcessingNode(processingWorklet, 'AudioRuntime');
+
+    // Connect nodes
+    workletSourceNodeRef.current.connect(workletProcessingNodeRef.current);
+    workletProcessingNodeRef.current.connect(workletNodeRef.current);
+    workletSourceNodeRef.current.connect(workletNodeRef.current);
     workletNodeRef.current.connect(aCtxRef.current.destination);
 
-    recorderRef.current.connect(recorderAdapterRef.current);
-    recorderRef.current.start();
-    console.log("Recording started");
-
+    workletSourceNodeRef.current.start();
     if (aCtxRef.current.state === 'suspended') {
       aCtxRef.current.resume();
     }
   }
 
   const stop = () => {
-    if (!recorderRef.current) {
-      console.error("Recorder is not initialized");
-      return;
-    }
-    recorderRef.current.stop();
-    recorderAdapterRef.current = null;
-    aCtxRef.current = null;
     console.log("Recording stopped");
+    workletSourceNodeRef.current?.stop();
     bar0.value = withSpring(0, { damping: 20, stiffness: 100 });
     bar1.value = withSpring(0, { damping: 20, stiffness: 100 });
     bar2.value = withSpring(0, { damping: 20, stiffness: 100 });

apps/fabric-example/ios/Podfile.lock

@@ -2342,7 +2342,7 @@ PODS:
     - React-perflogger (= 0.81.0)
     - React-utils (= 0.81.0)
     - SocketRocket
-  - RNAudioAPI (0.8.3):
+  - RNAudioAPI (0.9.0):
     - boost
     - DoubleConversion
     - fast_float
@@ -2368,10 +2368,10 @@ PODS:
     - ReactCodegen
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
-    - RNAudioAPI/audioapi (= 0.8.3)
+    - RNAudioAPI/audioapi (= 0.9.0)
     - SocketRocket
     - Yoga
-  - RNAudioAPI/audioapi (0.8.3):
+  - RNAudioAPI/audioapi (0.9.0):
     - boost
     - DoubleConversion
     - fast_float
@@ -2397,10 +2397,10 @@ PODS:
     - ReactCodegen
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
-    - RNAudioAPI/audioapi/ios (= 0.8.3)
+    - RNAudioAPI/audioapi/ios (= 0.9.0)
     - SocketRocket
     - Yoga
-  - RNAudioAPI/audioapi/ios (0.8.3):
+  - RNAudioAPI/audioapi/ios (0.9.0):
     - boost
     - DoubleConversion
     - fast_float

packages/audiodocs/docs/core/base-audio-context.mdx

@@ -67,6 +67,7 @@ Creates [`WorkletNode`](/docs/worklets/worklet-node).
 | `worklet` | `(Array<Float32Array>, number) => void` | The worklet to be executed. |
 | `bufferLength` | `number` | The size of the buffer that will be passed to the worklet on each call. |
 | `inputChannelCount` | `number` | The number of channels that the node expects as input (it will get min(expected, provided)). |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
 
 #### Errors
 
@@ -78,6 +79,40 @@ Creates [`WorkletNode`](/docs/worklets/worklet-node).
 
 #### Returns `WorkletNode`.
 
+### `createWorkletSourceNode` <MobileOnly />
+
+Creates [`WorkletSourceNode`](/docs/worklets/worklet-source-node).
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `worklet` | `(Array<Float32Array>, number, number, number) => void` | The worklet to be executed. |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
+
+#### Errors
+
+| Error type | Description |
+| :---: | :---- |
+| `Error` | `react-native-worklet` is not found as dependency. |
+
+#### Returns `WorkletSourceNode`.
+
+### `createWorkletProcessingNode` <MobileOnly />
+
+Creates [`WorkletProcessingNode`](/docs/worklets/worklet-processing-node).
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `worklet` | `(Array<Float32Array>, Array<Float32Array>, number, number) => void` | The worklet to be executed. |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
+
+#### Errors
+
+| Error type | Description |
+| :---: | :---- |
+| `Error` | `react-native-worklet` is not found as dependency. |
+
+#### Returns `WorkletProcessingNode`.
+
 ### `createBuffer`
 
 Creates [`AudioBuffer`](/docs/sources/audio-buffer).

packages/audiodocs/docs/worklets/introduction.mdx

@@ -26,9 +26,27 @@ Simply put, a worklet is a piece of code that can be executed on a runtime diffe
 
 ## What kind of worklets are used in react-native-audio-api?
 
-All of our worklets are currently executed on the UI runtime provided by the `RNWorklets` library.
+We support two types of worklet runtimes, each optimized for different use cases:
 
-This allows the use of Reanimated utilities and features inside the worklets. Our main goal is to enable seamless integration with the UI - for example, creating animations from audio data.
+### UIRuntime
+Worklets executed on the UI runtime provided by the `RNWorklets` library. This allows the use of Reanimated utilities and features inside the worklets. The main goal is to enable seamless integration with the UI - for example, creating animations from audio data.
+
+**Use UIRuntime when:**
+- You need to update UI elements from audio data
+- Creating visualizations or animations based on audio
+- Integrating with Reanimated shared values
+- Performance is less critical than UI responsiveness
+
+### AudioRuntime
+Worklets executed on the audio rendering thread for maximum performance and minimal latency. This runtime is optimized for real-time audio processing where timing is critical.
+
+**Use AudioRuntime when:**
+- Performance and low latency are crucial
+- Processing audio in real-time without dropouts
+- Generating audio with precise timing
+- Audio processing doesn't need to interact with UI
+
+You can specify the runtime type when creating worklet nodes using the `workletRuntime` parameter.
 
 ## How to use worklets in react-native-audio-api mindfully?
 

packages/audiodocs/docs/worklets/worklet-node.mdx

@@ -13,13 +13,13 @@ This node lets you execute a worklet on the UI thread. bufferLength specifies th
 
 ## Constructor
 
-[`BaseAudioContext.createWorkletNode(worklet, bufferLength, inputChannelCount)`](/docs/core/base-audio-context#createworkletnode-)
+[`BaseAudioContext.createWorkletNode(worklet, bufferLength, inputChannelCount, workletRuntime)`](/docs/core/base-audio-context#createworkletnode-)
 
 ## Example
 ```tsx
 import { AudioContext, AudioRecorder } from 'react-native-audio-api';
 
-/// This example shows how we can use a WorkletNode to process microphone audio data in real-time.
+// This example shows how we can use a WorkletNode to process microphone audio data in real-time.
 function App() {
     const recorder = new AudioRecorder({
       sampleRate: 16000,
@@ -34,7 +34,7 @@ function App() {
         // !IMPORTANT: here you can only read audio data any modifications will not be reflected in the audio output of this node
         // !VERY IMPORTANT: please read the Known Issue section below
     };
-    const workletNode = audioContext.createWorkletNode(worklet, 1024, 2);
+    const workletNode = audioContext.createWorkletNode(worklet, 1024, 2, 'UIRuntime');
     const adapterNode = audioContext.createRecorderAdapter();
 
     adapterNode.connect(workletNode);
@@ -45,13 +45,13 @@ function App() {
 ```
 
 ## Properties
-It has no own properties but inherits from `AudioNode`.
+It has no own properties but inherits from [`AudioNode`](/docs/core/audio-node).
 
 ## Methods
-It has no own methods but inherits from `AudioNode`.
+It has no own methods but inherits from [`AudioNode`](/docs/core/audio-node).
 
 ## Known Issue
-It might happen that the worklet side effect is not visible on the UI. For example you have some animated style which depends on some shared value modified in the worklet.
+It might happen that the worklet side effect is not visible on the UI (when you are using UIRuntime kind). For example you have some animated style which depends on some shared value modified in the worklet.
 This is happening because microtask queue is not always being flushed properly, bla bla bla...
 
 To workaround this issue just add this line at the end of your worklet callback function: