feat: custom worklet processor node (#704)

* feat: made context hold multiple worklet runtimes

* chore: renamed worklet runner

* feat: implemented worklet source node

* chore: some leftover

* docs: updated documentation

* docs: neatpicks

* feat: implemented worklet processing node

* fix: fixed tests

* fix: replaced magic numbers

* fix: requested changes

* fix: fixed newer tests

Author: poneciak57

apps/common-app/src/examples/Worklets/Worklets.tsx

@@ -3,9 +3,9 @@ import { Text, View, StyleSheet } from 'react-native';
 import {
   AudioContext,
   AudioManager,
-  AudioRecorder,
-  RecorderAdapterNode,
-  WorkletNode
+  WorkletNode,
+  WorkletProcessingNode,
+  WorkletSourceNode
 } from 'react-native-audio-api';
 import { Container, Button } from "../../components";
 import { Extrapolation, useSharedValue } from "react-native-reanimated";
@@ -16,13 +16,12 @@ import Animated, {
 } from "react-native-reanimated";
 import { colors } from "../../styles";
 
-
 function Worklets() {
   const SAMPLE_RATE = 44100;
-  const recorderRef = useRef<AudioRecorder | null>(null);
   const aCtxRef = useRef<AudioContext | null>(null);
-  const recorderAdapterRef = useRef<RecorderAdapterNode | null>(null);
   const workletNodeRef = useRef<WorkletNode | null>(null);
+  const workletProcessingNodeRef = useRef<WorkletProcessingNode | null>(null);
+  const workletSourceNodeRef = useRef<WorkletSourceNode | null>(null);
 
   const bar0 = useSharedValue(0);
   const bar1 = useSharedValue(0);
@@ -36,65 +35,71 @@ function Worklets() {
       iosMode: 'spokenAudio',
       iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'],
     });
-
-    AudioManager.requestRecordingPermissions();
-    recorderRef.current = new AudioRecorder({
-      sampleRate: SAMPLE_RATE,
-      bufferLengthInSamples: 1024,
-    });
   }, []);
 
   const start = () => {
-    if (!recorderRef.current) {
-      console.error("Recorder is not initialized");
-      return;
+
+    const processingWorklet = (inputAudioData: Array<Float32Array>, outputAudioData: Array<Float32Array>, framesToProcess: number, currentTime: number) => {
+      'worklet';
+      const gain = 0.5;
+      for (let channel = 0; channel < inputAudioData.length; channel++) {
+        const inputChannelData = inputAudioData[channel];
+        const outputChannelData = outputAudioData[channel];
+        for (let i = 0; i < framesToProcess; i++) {
+          outputChannelData[i] = inputChannelData[i] * gain;
+        }
+      }
+    };
+
+    const sourceWorklet = (audioData: Array<Float32Array>, framesToProcess: number, currentTime: number, startOffset: number) => {
+      'worklet';
+      for (let i = 0; i < audioData.length; i++) {
+        for (let j = 0; j < framesToProcess; j++) {
+          audioData[i][j] = Math.sin((currentTime + startOffset + j));
+        }
+      }
     }
 
     const worklet = (audioData: Array<Float32Array>, inputChannelCount: number) => {
       'worklet';
 
-
       // Calculates RMS amplitude
       let sum = 0;
       for (let i = 0; i < audioData.length; i++) {
         sum += audioData[0][i] * audioData[0][i];
       }
       const rms = Math.sqrt(sum / audioData[0].length);
-      const scaledAmplitude = Math.min(rms * 1000, 1);
+      const scaledAmplitude = Math.min(rms * 10, 1);
 
-      console.log(`RMS: ${rms}, Scaled: ${scaledAmplitude}`);
+      // console.log(`RMS: ${rms}, Scaled: ${scaledAmplitude}`);
 
       bar0.value = withSpring(bar1.value, { damping: 20, stiffness: 150 });
       bar1.value = withSpring(bar2.value, { damping: 20, stiffness: 150 });
       bar3.value = withSpring(bar2.value, { damping: 20, stiffness: 150 });
       bar4.value = withSpring(bar3.value, { damping: 20, stiffness: 150 });
-      bar2.value = withSpring(scaledAmplitude, { damping: 20, stiffness: 200 });
+      bar2.value = withSpring(scaledAmplitude, { damping: 20, stiffness: 150 });
     };
 
     aCtxRef.current = new AudioContext({ sampleRate: SAMPLE_RATE });
-    recorderAdapterRef.current = aCtxRef.current.createRecorderAdapter();
-    workletNodeRef.current = aCtxRef.current.createWorkletNode(worklet, 512, 1);
-    recorderAdapterRef.current.connect(workletNodeRef.current);
+    workletSourceNodeRef.current = aCtxRef.current.createWorkletSourceNode(sourceWorklet, 'AudioRuntime');
+    workletNodeRef.current = aCtxRef.current.createWorkletNode(worklet, 256, 1, 'UIRuntime');
+    workletProcessingNodeRef.current = aCtxRef.current.createWorkletProcessingNode(processingWorklet, 'AudioRuntime');
+
+    // Connect nodes
+    workletSourceNodeRef.current.connect(workletProcessingNodeRef.current);
+    workletProcessingNodeRef.current.connect(workletNodeRef.current);
+    workletSourceNodeRef.current.connect(workletNodeRef.current);
     workletNodeRef.current.connect(aCtxRef.current.destination);
 
-    recorderRef.current.connect(recorderAdapterRef.current);
-    recorderRef.current.start();
-    console.log("Recording started");
-
+    workletSourceNodeRef.current.start();
     if (aCtxRef.current.state === 'suspended') {
       aCtxRef.current.resume();
     }
   }
 
   const stop = () => {
-    if (!recorderRef.current) {
-      console.error("Recorder is not initialized");
-      return;
-    }
-    recorderRef.current.stop();
-    recorderAdapterRef.current = null;
-    aCtxRef.current = null;
     console.log("Recording stopped");
+    workletSourceNodeRef.current?.stop();
     bar0.value = withSpring(0, { damping: 20, stiffness: 100 });
     bar1.value = withSpring(0, { damping: 20, stiffness: 100 });
     bar2.value = withSpring(0, { damping: 20, stiffness: 100 });

packages/audiodocs/docs/core/base-audio-context.mdx

@@ -67,6 +67,7 @@ Creates [`WorkletNode`](/docs/worklets/worklet-node).
 | `worklet` | `(Array<Float32Array>, number) => void` | The worklet to be executed. |
 | `bufferLength` | `number` | The size of the buffer that will be passed to the worklet on each call. |
 | `inputChannelCount` | `number` | The number of channels that the node expects as input (it will get min(expected, provided)). |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
 
 #### Errors
 
@@ -78,6 +79,40 @@ Creates [`WorkletNode`](/docs/worklets/worklet-node).
 
 #### Returns `WorkletNode`.
 
+### `createWorkletSourceNode` <MobileOnly />
+
+Creates [`WorkletSourceNode`](/docs/worklets/worklet-source-node).
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `worklet` | `(Array<Float32Array>, number, number, number) => void` | The worklet to be executed. |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
+
+#### Errors
+
+| Error type | Description |
+| :---: | :---- |
+| `Error` | `react-native-worklet` is not found as dependency. |
+
+#### Returns `WorkletSourceNode`.
+
+### `createWorkletProcessingNode` <MobileOnly />
+
+Creates [`WorkletProcessingNode`](/docs/worklets/worklet-processing-node).
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `worklet` | `(Array<Float32Array>, Array<Float32Array>, number, number) => void` | The worklet to be executed. |
+| `workletRuntime` | `AudioWorkletRuntime` | The kind of runtime to use for the worklet. See [worklet runtimes](/docs/worklets/worklets-introduction#what-kind-of-worklets-are-used-in-react-native-audio-api) for details. |
+
+#### Errors
+
+| Error type | Description |
+| :---: | :---- |
+| `Error` | `react-native-worklet` is not found as dependency. |
+
+#### Returns `WorkletProcessingNode`.
+
 ### `createBuffer`
 
 Creates [`AudioBuffer`](/docs/sources/audio-buffer).

packages/audiodocs/docs/worklets/introduction.mdx

@@ -26,9 +26,27 @@ Simply put, a worklet is a piece of code that can be executed on a runtime diffe
 
 ## What kind of worklets are used in react-native-audio-api?
 
-All of our worklets are currently executed on the UI runtime provided by the `RNWorklets` library.
+We support two types of worklet runtimes, each optimized for different use cases:
 
-This allows the use of Reanimated utilities and features inside the worklets. Our main goal is to enable seamless integration with the UI - for example, creating animations from audio data.
+### UIRuntime
+Worklets executed on the UI runtime provided by the `RNWorklets` library. This allows the use of Reanimated utilities and features inside the worklets. The main goal is to enable seamless integration with the UI - for example, creating animations from audio data.
+
+**Use UIRuntime when:**
+- You need to update UI elements from audio data
+- Creating visualizations or animations based on audio
+- Integrating with Reanimated shared values
+- Performance is less critical than UI responsiveness
+
+### AudioRuntime
+Worklets executed on the audio rendering thread for maximum performance and minimal latency. This runtime is optimized for real-time audio processing where timing is critical.
+
+**Use AudioRuntime when:**
+- Performance and low latency are crucial
+- Processing audio in real-time without dropouts
+- Generating audio with precise timing
+- Audio processing doesn't need to interact with UI
+
+You can specify the runtime type when creating worklet nodes using the `workletRuntime` parameter.
 
 ## How to use worklets in react-native-audio-api mindfully?
 

packages/audiodocs/docs/worklets/worklet-node.mdx

@@ -13,13 +13,13 @@ This node lets you execute a worklet on the UI thread. bufferLength specifies th
 
 ## Constructor
 
-[`BaseAudioContext.createWorkletNode(worklet, bufferLength, inputChannelCount)`](/docs/core/base-audio-context#createworkletnode-)
+[`BaseAudioContext.createWorkletNode(worklet, bufferLength, inputChannelCount, workletRuntime)`](/docs/core/base-audio-context#createworkletnode-)
 
 ## Example
 ```tsx
 import { AudioContext, AudioRecorder } from 'react-native-audio-api';
 
-/// This example shows how we can use a WorkletNode to process microphone audio data in real-time.
+// This example shows how we can use a WorkletNode to process microphone audio data in real-time.
 function App() {
     const recorder = new AudioRecorder({
       sampleRate: 16000,
@@ -34,7 +34,7 @@ function App() {
         // !IMPORTANT: here you can only read audio data any modifications will not be reflected in the audio output of this node
         // !VERY IMPORTANT: please read the Known Issue section below
     };
-    const workletNode = audioContext.createWorkletNode(worklet, 1024, 2);
+    const workletNode = audioContext.createWorkletNode(worklet, 1024, 2, 'UIRuntime');
     const adapterNode = audioContext.createRecorderAdapter();
 
     adapterNode.connect(workletNode);
@@ -45,13 +45,13 @@ function App() {
 ```
 
 ## Properties
-It has no own properties but inherits from `AudioNode`.
+It has no own properties but inherits from [`AudioNode`](/docs/core/audio-node).
 
 ## Methods
-It has no own methods but inherits from `AudioNode`.
+It has no own methods but inherits from [`AudioNode`](/docs/core/audio-node).
 
 ## Known Issue
-It might happen that the worklet side effect is not visible on the UI. For example you have some animated style which depends on some shared value modified in the worklet.
+It might happen that the worklet side effect is not visible on the UI (when you are using UIRuntime kind). For example you have some animated style which depends on some shared value modified in the worklet.
 This is happening because microtask queue is not always being flushed properly, bla bla bla...
 
 To workaround this issue just add this line at the end of your worklet callback function:

packages/audiodocs/docs/worklets/worklet-processing-node.mdx

@@ -0,0 +1,143 @@
+---
+sidebar_position: 4
+---
+
+import { ReadOnly } from '@site/src/components/Badges';
+
+# WorkletProcessingNode
+
+The `WorkletProcessingNode` interface represents a node in the audio processing graph that can process audio using a worklet function. Unlike [`WorkletNode`](/docs/worklets/worklet-node) which only provides read-only access to audio data, `WorkletProcessingNode` allows you to modify the audio signal by providing both input and output buffers.
+
+This node lets you execute a worklet that receives input audio data and produces output audio data, making it perfect for creating custom audio effects, filters, and processors. The worklet processes the exact number of frames provided by the audio system in each call.
+
+For more information about worklets, see our [Introduction to worklets](/docs/worklets/worklets-introduction).
+
+## Constructor
+
+[`BaseAudioContext.createWorkletProcessingNode(worklet, workletRuntime)`](/docs/core/base-audio-context#createworkletprocessingnode-)
+
+## Example
+
+```tsx
+import { AudioContext, AudioRecorder } from 'react-native-audio-api';
+
+// This example shows how to create a simple gain effect using WorkletProcessingNode
+function App() {
+    const recorder = new AudioRecorder({
+      sampleRate: 16000,
+      bufferLengthInSamples: 16000,
+    });
+
+    const audioContext = new AudioContext({ sampleRate: 16000 });
+
+    // Create a simple gain worklet that multiplies the input by a gain value
+    const gainWorklet = (
+        inputData: Array<Float32Array>,
+        outputData: Array<Float32Array>,
+        framesToProcess: number,
+        currentTime: number
+    ) => {
+        'worklet';
+        const gain = 0.5; // 50% volume
+
+        for (let ch = 0; ch < inputData.length; ch++) {
+            const input = inputData[ch];
+            const output = outputData[ch];
+
+            for (let i = 0; i < framesToProcess; i++) {
+                output[i] = input[i] * gain;
+            }
+        }
+    };
+
+    const workletProcessingNode = audioContext.createWorkletProcessingNode(
+        gainWorklet,
+        'AudioRuntime'
+    );
+    const adapterNode = audioContext.createRecorderAdapter();
+
+    adapterNode.connect(workletProcessingNode);
+    workletProcessingNode.connect(audioContext.destination);
+    recorder.connect(adapterNode);
+    recorder.start();
+}
+}
+```
+
+## Worklet Parameters Explanation
+
+The worklet function receives four parameters:
+
+### `inputData: Array<Float32Array>`
+A two-dimensional array where:
+- First dimension represents the audio channel (0 = left, 1 = right for stereo)
+- Second dimension contains the input audio samples for that channel
+- You should **read** from these buffers to get the input audio data
+- The length of each `Float32Array` equals the `framesToProcess` parameter
+
+### `outputData: Array<Float32Array>`
+A two-dimensional array where:
+- First dimension represents the audio channel (0 = left, 1 = right for stereo)
+- Second dimension contains the output audio samples for that channel
+- You must **write** to these buffers to produce the processed audio output
+- The length of each `Float32Array` equals the `framesToProcess` parameter
+
+### `framesToProcess: number`
+The number of audio samples to process in this call. This determines how many samples you need to process in each channel's buffer. This value will be at most 128.
+
+### `currentTime: number`
+The current audio context time in seconds when this worklet call begins. This represents the absolute time since the audio context was created.
+
+## Audio Processing Pattern
+
+A typical WorkletProcessingNode worklet follows this pattern:
+
+```tsx
+const audioProcessor = (
+    inputData: Array<Float32Array>,
+    outputData: Array<Float32Array>,
+    framesToProcess: number,
+    currentTime: number
+) => {
+    'worklet';
+
+    for (let channel = 0; channel < inputData.length; channel++) {
+        const input = inputData[channel];
+        const output = outputData[channel];
+
+        for (let sample = 0; sample < framesToProcess; sample++) {
+            // Process each sample
+            // Read from: input[sample]
+            // Write to: output[sample]
+            output[sample] = processAudioSample(input[sample]);
+        }
+    }
+};
+```
+
+## Properties
+
+It has no own properties but inherits from [`AudioNode`](/docs/core/audio-node).
+
+## Methods
+
+It has no own methods but inherits from [`AudioNode`](/docs/core/audio-node).
+
+## Performance Considerations
+
+Since `WorkletProcessingNode` processes audio in real-time, performance is critical:
+
+- Keep worklet functions lightweight and efficient
+- Avoid complex calculations that could cause audio dropouts
+- Process samples in-place when possible
+- Consider using lookup tables for expensive operations
+- Use `AudioRuntime` for better performance, `UIRuntime` for UI integration
+- Test on target devices to ensure smooth audio processing
+
+## Use Cases
+
+- **Audio Effects**: Reverb, delay, distortion, filters
+- **Audio Processing**: Compression, limiting, normalization
+- **Real-time Filters**: EQ, high-pass, low-pass, band-pass filters
+- **Custom Algorithms**: Noise reduction, pitch shifting, spectral processing
+- **Signal Analysis**: Feature extraction while passing audio through