Audio stretcher module

apps/common-app/src/examples/PlaybackSpeed/PlaybackSpeed.tsx

@@ -8,7 +8,7 @@ import React, {
 } from 'react';
 import { View, StyleSheet } from 'react-native';
 import { Container, Button, Spacer, Slider, Select } from '../../components';
-import { AudioContext } from 'react-native-audio-api';
+import { AudioContext, changePlaybackSpeed } from 'react-native-audio-api';
 import type { AudioBufferSourceNode } from 'react-native-audio-api';
 import {
   PCM_DATA,
@@ -49,7 +49,11 @@ const PlaybackSpeed: FC = () => {
     setIsLoading(true);
 
     try {
-      const buffer = await audioContext.decodeAudioData(PCM_DATA);
+      let buffer = await audioContext.decodeAudioData(PCM_DATA);
+      buffer = await changePlaybackSpeed(
+        buffer,
+        audioSettings.PSOLA ? playbackSpeed : 1
+      );
 
       const source = audioContext.createBufferSource({
         pitchCorrection: audioSettings.PSOLA

packages/audiodocs/docs/sources/audio-buffer.mdx

@@ -12,7 +12,7 @@ specific sample rate which is the quantity of frames that will play in one secon
 
 ![](/img/audioBuffer.png)
 
-It can be created from audio file using [`decodeAudioData`](/docs/utils/audio-decoder#decodeaudiodata) or from raw data using `constructor`.
+It can be created from audio file using [`decodeAudioData`](/docs/utils/decoding#decodeaudiodata) or from raw data using `constructor`.
 Once you have data in `AudioBuffer`, audio can be played by passing it to [`AudioBufferSourceNode`](audio-buffer-source-node).
 
 ## Constructor

packages/audiodocs/docs/utils/decoding.mdx

@@ -6,8 +6,8 @@ import { Optional, MobileOnly } from '@site/src/components/Badges';
 
 # Decoding
 
-You can decode audio data independently, without creating an AudioContext, using the exported functions [`decodeAudioData`](/docs/utils/audio-decoder#decodeaudiodata) and
-[`decodePCMInBase64`](/docs/utils/audio-decoder#decodepcminbase64).
+You can decode audio data independently, without creating an AudioContext, using the exported functions [`decodeAudioData`](/docs/utils/decoding#decodeaudiodata) and
+[`decodePCMInBase64`](/docs/utils/decoding#decodepcminbase64).
 
 If you already have an audio context, you can decode audio data directly using its [`decodeAudioData`](/docs/core/base-audio-context#decodeaudiodata) function;
 the decoded audio will then be automatically resampled to match the context's `sampleRate`.

packages/audiodocs/docs/utils/time-stretching.mdx

@@ -0,0 +1,39 @@
+---
+sidebar_position: 2
+---
+
+import { Optional, MobileOnly } from '@site/src/components/Badges';
+
+# Time stretching
+
+You can change the playback speed of an audio buffer independently, without creating an AudioContext, using the exported function [`changePlaybackSpeed`](/docs/utils/decoding#decodeaudiodata).
+
+### `changePlaybackSpeed`
+
+Changes the playback speed of an audio buffer.
+
+| Parameter | Type | Description |
+| :----: | :----: | :-------- |
+| `input` | `AudioBuffer` | The audio buffer whose playback speed you want to change. |
+| `playbackSpeed` | `number` | The factor by which to change the playback speed. Values between [1.0, 2.0] speed up playback, values between [0.5, 1.0] slow it down. |
+
+
+#### Returns `Promise<AudioBuffer>`.
+
+<details>
+<summary>Example usage</summary>
+```tsx
+const url = ... // url to an audio
+const sampleRate = 48000
+
+const buffer = await fetch(url)
+  .then((response) => response.arrayBuffer())
+  .then((arrayBuffer) => decodeAudioData(arrayBuffer, sampleRate))
+  .then((audioBuffer) => changePlaybackSpeed(audioBuffer, 1.25))
+  .catch((error) => {
+    console.error('Error decoding audio data source:', error);
+    return null;
+  });
+```
+</details>
+

packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/AudioStretcher.cpp

@@ -0,0 +1,74 @@
+#include <audioapi/core/sources/AudioBuffer.h>
+#include <audioapi/core/utils/AudioStretcher.h>
+#include <audioapi/libs/audio-stretch/stretch.h>
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/utils/AudioBus.h>
+#include <cstdint>
+
+namespace audioapi {
+
+std::vector<int16_t> AudioStretcher::castToInt16Buffer(AudioBuffer &buffer) {
+  const size_t numChannels = buffer.getNumberOfChannels();
+  const size_t numFrames = buffer.getLength();
+
+  std::vector<int16_t> int16Buffer(numFrames * numChannels);
+
+  for (size_t ch = 0; ch < numChannels; ++ch) {
+    const float *channelData = buffer.getChannelData(ch);
+    for (size_t i = 0; i < numFrames; ++i) {
+      int16Buffer[i * numChannels + ch] = floatToInt16(channelData[i]);
+    }
+  }
+
+  return int16Buffer;
+}
+
+std::shared_ptr<AudioBuffer> AudioStretcher::changePlaybackSpeed(
+    AudioBuffer buffer,
+    float playbackSpeed) {
+  const float sampleRate = buffer.getSampleRate();
+  const size_t outputChannels = buffer.getNumberOfChannels();
+  const size_t numFrames = buffer.getLength();
+
+  if (playbackSpeed == 1.0f) {
+    return std::make_shared<AudioBuffer>(buffer);
+  }
+
+  std::vector<int16_t> int16Buffer = castToInt16Buffer(buffer);
+
+  auto stretcher = stretch_init(
+      static_cast<int>(sampleRate / 333.0f),
+      static_cast<int>(sampleRate / 55.0f),
+      outputChannels,
+      0x1);
+
+  int maxOutputFrames = stretch_output_capacity(
+      stretcher, static_cast<int>(numFrames), 1 / playbackSpeed);
+  std::vector<int16_t> stretchedBuffer(maxOutputFrames * outputChannels);
+
+  int outputFrames = stretch_samples(
+      stretcher,
+      int16Buffer.data(),
+      static_cast<int>(numFrames),
+      stretchedBuffer.data(),
+      1 / playbackSpeed);
+
+  outputFrames +=
+      stretch_flush(stretcher, stretchedBuffer.data() + (outputFrames));
+  stretchedBuffer.resize(outputFrames * outputChannels);
+  stretch_deinit(stretcher);
+
+  auto audioBus =
+      std::make_shared<AudioBus>(outputFrames, outputChannels, sampleRate);
+
+  for (int ch = 0; ch < outputChannels; ++ch) {
+    auto channelData = audioBus->getChannel(ch)->getData();
+    for (int i = 0; i < outputFrames; ++i) {
+      channelData[i] = int16ToFloat(stretchedBuffer[i * outputChannels + ch]);
+    }
+  }
+
+  return std::make_shared<AudioBuffer>(audioBus);
+}
+
+} // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h

@@ -4,6 +4,7 @@
 #include <audioapi/HostObjects/OfflineAudioContextHostObject.h>
 #include <audioapi/HostObjects/inputs/AudioRecorderHostObject.h>
 #include <audioapi/HostObjects/utils/AudioDecoderHostObject.h>
+#include <audioapi/HostObjects/utils/AudioStretcherHostObject.h>
 #include <audioapi/core/AudioContext.h>
 #include <audioapi/core/OfflineAudioContext.h>
 #include <audioapi/core/inputs/AudioRecorder.h>
@@ -38,6 +39,8 @@ class AudioAPIModuleInstaller {
         jsiRuntime, jsCallInvoker, audioEventHandlerRegistry, workletRunner);
     auto createAudioDecoder =
         getCreateAudioDecoderFunction(jsiRuntime, jsCallInvoker);
+    auto createAudioStretcher =
+        getCreateAudioStretcherFunction(jsiRuntime, jsCallInvoker);
 
     jsiRuntime->global().setProperty(
         *jsiRuntime, "createAudioContext", createAudioContext);
@@ -47,6 +50,8 @@ class AudioAPIModuleInstaller {
         *jsiRuntime, "createOfflineAudioContext", createOfflineAudioContext);
     jsiRuntime->global().setProperty(
         *jsiRuntime, "createAudioDecoder", createAudioDecoder);
+    jsiRuntime->global().setProperty(
+        *jsiRuntime, "createAudioStretcher", createAudioStretcher);
 
     auto audioEventHandlerRegistryHostObject =
         std::make_shared<AudioEventHandlerRegistryHostObject>(
@@ -174,6 +179,26 @@ class AudioAPIModuleInstaller {
               runtime, audioDecoderHostObject);
         });
   }
+
+  static jsi::Function getCreateAudioStretcherFunction(
+      jsi::Runtime *jsiRuntime,
+      const std::shared_ptr<react::CallInvoker> &jsCallInvoker) {
+    return jsi::Function::createFromHostFunction(
+        *jsiRuntime,
+        jsi::PropNameID::forAscii(*jsiRuntime, "createAudioStretcher"),
+        0,
+        [jsCallInvoker](
+            jsi::Runtime &runtime,
+            const jsi::Value &thisValue,
+            const jsi::Value *args,
+            size_t count) -> jsi::Value {
+          auto audioStretcherHostObject =
+              std::make_shared<AudioStretcherHostObject>(
+                  &runtime, jsCallInvoker);
+          return jsi::Object::createFromHostObject(
+              runtime, audioStretcherHostObject);
+        });
+  }
 };
 
 } // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioStretcherHostObject.cpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <audioapi/HostObjects/sources/AudioBufferHostObject.h>
+#include <audioapi/HostObjects/utils/AudioStretcherHostObject.h>
+#include <audioapi/core/utils/AudioStretcher.h>
+#include <audioapi/jsi/JsiPromise.h>
+
+#include <jsi/jsi.h>
+#include <memory>
+#include <string>
+#include <thread>
+#include <utility>
+
+namespace audioapi {
+
+AudioStretcherHostObject::AudioStretcherHostObject(
+    jsi::Runtime *runtime,
+    const std::shared_ptr<react::CallInvoker> &callInvoker) {
+  promiseVendor_ = std::make_shared<PromiseVendor>(runtime, callInvoker);
+  stretcher_ = std::make_shared<AudioStretcher>();
+  addFunctions(
+      JSI_EXPORT_FUNCTION(AudioStretcherHostObject, changePlaybackSpeed));
+}
+
+JSI_HOST_FUNCTION_IMPL(AudioStretcherHostObject, changePlaybackSpeed) {
+  auto audioBuffer =
+      args[0].getObject(runtime).asHostObject<AudioBufferHostObject>(runtime);
+  auto playbackSpeed = static_cast<float>(args[1].asNumber());
+
+  auto promise = promiseVendor_->createPromise(
+      [this, audioBuffer, playbackSpeed](std::shared_ptr<Promise> promise) {
+        std::thread([this,
+                     audioBuffer,
+                     playbackSpeed,
+                     promise = std::move(promise)]() {
+          auto result = stretcher_->changePlaybackSpeed(
+              *audioBuffer->audioBuffer_, playbackSpeed);
+
+          if (!result) {
+            promise->reject("Failed to change audio playback speed.");
+            return;
+          }
+
+          auto audioBufferHostObject =
+              std::make_shared<AudioBufferHostObject>(result);
+
+          promise->resolve([audioBufferHostObject = std::move(
+                                audioBufferHostObject)](jsi::Runtime &runtime) {
+            auto jsiObject = jsi::Object::createFromHostObject(
+                runtime, audioBufferHostObject);
+            jsiObject.setExternalMemoryPressure(
+                runtime, audioBufferHostObject->getSizeInBytes());
+            return jsiObject;
+          });
+        }).detach();
+      });
+  return promise;
+}
+
+} // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioStretcherHostObject.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <audioapi/HostObjects/sources/AudioBufferHostObject.h>
+#include <audioapi/core/utils/AudioStretcher.h>
+#include <audioapi/jsi/JsiPromise.h>
+
+#include <jsi/jsi.h>
+#include <memory>
+#include <string>
+#include <thread>
+#include <utility>
+
+namespace audioapi {
+using namespace facebook;
+
+class AudioStretcherHostObject : public JsiHostObject {
+ public:
+  explicit AudioStretcherHostObject(
+      jsi::Runtime *runtime,
+      const std::shared_ptr<react::CallInvoker> &callInvoker);
+  JSI_HOST_FUNCTION_DECL(changePlaybackSpeed);
+
+ private:
+  std::shared_ptr<AudioStretcher> stretcher_;
+  std::shared_ptr<PromiseVendor> promiseVendor_;
+};
+} // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioDecoder.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <audioapi/core/types/AudioFormat.h>
-#include <audioapi/libs/audio-stretch/stretch.h>
 #include <audioapi/libs/miniaudio/miniaudio.h>
 #include <algorithm>
 #include <cstring>

packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioStretcher.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+
+namespace audioapi {
+
+class AudioBus;
+class AudioBuffer;
+
+class AudioStretcher {
+ public:
+  explicit AudioStretcher() {}
+
+  [[nodiscard]] static std::shared_ptr<AudioBuffer> changePlaybackSpeed(
+      AudioBuffer buffer,
+      float playbackSpeed);
+
+ private:
+  float sampleRate_;
+
+  static std::vector<int16_t> castToInt16Buffer(AudioBuffer &buffer);
+
+  [[nodiscard]] static inline int16_t floatToInt16(float sample) {
+    return static_cast<int16_t>(sample * 32768.0f);
+  }
+  [[nodiscard]] static inline float int16ToFloat(int16_t sample) {
+    return static_cast<float>(sample) / 32768.0f;
+  }
+};
+
+} // namespace audioapi