diff --git a/apps/common-app/src/examples/PlaybackSpeed/PlaybackSpeed.tsx b/apps/common-app/src/examples/PlaybackSpeed/PlaybackSpeed.tsx index 17b295d13..81a5e8658 100644 --- a/apps/common-app/src/examples/PlaybackSpeed/PlaybackSpeed.tsx +++ b/apps/common-app/src/examples/PlaybackSpeed/PlaybackSpeed.tsx @@ -49,9 +49,11 @@ const PlaybackSpeed: FC = () => { setIsLoading(true); try { - const buffer = await audioContext.decodePCMInBase64Data( + const buffer = await audioContext.decodePCMInBase64( PCM_DATA, - audioSettings.PSOLA ? playbackSpeed : 1 + 48000, + 1, + true ); const source = audioContext.createBufferSource({ diff --git a/apps/fabric-example/ios/Podfile.lock b/apps/fabric-example/ios/Podfile.lock index d836189ff..27581fcc1 100644 --- a/apps/fabric-example/ios/Podfile.lock +++ b/apps/fabric-example/ios/Podfile.lock @@ -3081,7 +3081,7 @@ SPEC CHECKSUMS: ReactAppDependencyProvider: c91900fa724baee992f01c05eeb4c9e01a807f78 ReactCodegen: 8125d6ee06ea06f48f156cbddec5c2ca576d62e6 ReactCommon: 116d6ee71679243698620d8cd9a9042541e44aa6 - RNAudioAPI: 56c615503d1c8bf8a936fc11fc2ff325b1fc7459 + RNAudioAPI: e08a4157527a2e87879a7bb61880276a0dfc77f6 RNGestureHandler: 3a73f098d74712952870e948b3d9cf7b6cae9961 RNReanimated: a035264789d1f64cb5adba7085d6aac6e9ec70a7 RNScreens: 6ced6ae8a526512a6eef6e28c2286e1fc2d378c3 diff --git a/packages/audiodocs/docs/core/base-audio-context.mdx b/packages/audiodocs/docs/core/base-audio-context.mdx index a120dfc91..0c8c20311 100644 --- a/packages/audiodocs/docs/core/base-audio-context.mdx +++ b/packages/audiodocs/docs/core/base-audio-context.mdx @@ -209,20 +209,51 @@ Creates [`BiquadFilterNode`](/docs/effects/biquad-filter-node). :::caution Supported file formats: -- mp3 -- wav +- aac - flac -- opus -- ogg - m4a -- aac +- mp3 - mp4 +- ogg +- opus +- wav ::: ### `decodeAudioData` +Decodes audio data from either a file path or an ArrayBuffer. The optional `sampleRate` parameter lets you resample the decoded audio. +If not provided, the audio will be automatically resampled to match the audio context's `sampleRate`. + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterTypeDescription
inputArrayBufferArrayBuffer with audio data.
stringPath to audio file located on the device.
sampleRatenumberTarget sample rate for the decoded audio.
+ +#### Returns `Promise`. +
-Example +Example decoding with memory block ```tsx const url = ... // url to an audio @@ -236,16 +267,6 @@ const buffer = await fetch(url) ```
-Decodes audio data. It decodes with in memory audio data block. - -| Parameters | Type | Description | -| :---: | :---: | :---- | -| `arrayBuffer` | `ArrayBuffer` | ArrayBuffer with audio data. | - -#### Returns `Promise`. - -### `decodeAudioDataSource` -
Example using expo-asset library ```tsx @@ -257,38 +278,35 @@ const buffer = await Asset.fromModule(require('@/assets/music/example.mp3')) if (!asset.localUri) { throw new Error('Failed to load audio asset'); } - return this.audioContext.decodeAudioDataSource(asset.localUri); + return this.audioContext.decodeAudioData(asset.localUri); }) ```
-Decodes audio data file. +### `decodePCMInBase64` -| Parameters | Type | Description | -| :---: | :---: | :---- | -| `sourcePath` | `string` | Path to audio file located on the device. | +Decodes base64-encoded PCM audio data. -#### Returns `Promise`. +#### Parameters -### `decodePCMInBase64Data` +| Parameter | Type | Description | +|-----------|------|-------------| +| `base64String` | `string` | Base64-encoded PCM audio data. | +| `inputSampleRate` | `number` | Sample rate of the input PCM data. | +| `inputChannelCount` | `number` | Number of channels in the input PCM data. | +| `isInterleaved` | `boolean` | Whether the PCM data is interleaved. Default is `true`. | + +#### Returns `Promise`
-Example +Example decoding with data in base64 format ```tsx const data = ... // data encoded in base64 string -const buffer = await this.audioContext.decodePCMInBase64Data(data); +// data is not interleaved (Channel1, Channel1, ..., Channel2, Channel2, ...) +const buffer = await this.audioContext.decodeAudioData(data, 4800, 2, false); ```
-Decodes audio data. It decodes with PCM data in Base64. - -| Parameters | Type | Description | -| :---: | :---: | :---- | -| `base64` | `string` | Base64 string with audio data. | -| `playbackRate` | `number` | Number that represents audio speed, which will be applied during decoding. | - -#### Returns `Promise`. - ## Remarks #### `currentTime` diff --git a/packages/audiodocs/docs/other/_category_.json b/packages/audiodocs/docs/other/_category_.json index 27d4146a5..114020593 100644 --- a/packages/audiodocs/docs/other/_category_.json +++ b/packages/audiodocs/docs/other/_category_.json @@ -1,6 +1,6 @@ { "label": "Other", - "position": 10, + "position": 11, "link": { "type": "generated-index" } diff --git a/packages/audiodocs/docs/sources/audio-buffer.mdx b/packages/audiodocs/docs/sources/audio-buffer.mdx index b050a54c2..e8a212f99 100644 --- a/packages/audiodocs/docs/sources/audio-buffer.mdx +++ b/packages/audiodocs/docs/sources/audio-buffer.mdx @@ -12,7 +12,7 @@ specific sample rate which is the quantity of frames that will play in one secon ![](/img/audioBuffer.png) -It can be created from audio file using [`decodeAudioData`](/docs/core/base-audio-context#decodeaudiodata), [`decodeAudioDataSource`](/docs/core/base-audio-context#decodeaudiodatasource), [`decodePCMInBase64Data`](/docs/core/base-audio-context#decodepcminbase64data-) or from raw data using `constructor`. +It can be created from audio file using [`decodeAudioData`](/docs/utils/audio-decoder#decodeaudiodata) or from raw data using `constructor`. Once you have data in `AudioBuffer`, audio can be played by passing it to [`AudioBufferSourceNode`](audio-buffer-source-node). ## Constructor diff --git a/packages/audiodocs/docs/utils/_category_.json b/packages/audiodocs/docs/utils/_category_.json new file mode 100644 index 000000000..7266f570f --- /dev/null +++ b/packages/audiodocs/docs/utils/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Utils", + "position": 10, + "link": { + "type": "generated-index" + } + } diff --git a/packages/audiodocs/docs/utils/decoding.mdx b/packages/audiodocs/docs/utils/decoding.mdx new file mode 100644 index 000000000..ee9d23434 --- /dev/null +++ b/packages/audiodocs/docs/utils/decoding.mdx @@ -0,0 +1,116 @@ +--- +sidebar_position: 1 +--- + +import { Optional, MobileOnly } from '@site/src/components/Badges'; + +# Decoding + +You can decode audio data independently, without creating an AudioContext, using the exported functions [`decodeAudioData`](/docs/utils/audio-decoder#decodeaudiodata) and +[`decodePCMInBase64`](/docs/utils/audio-decoder#decodepcminbase64). + +If you already have an audio context, you can decode audio data directly using its [`decodeAudioData`](/docs/core/base-audio-context#decodeaudiodata) function; +the decoded audio will then be automatically resampled to match the context's `sampleRate`. + +:::caution +Supported file formats: +- aac +- flac +- m4a +- mp3 +- mp4 +- ogg +- opus +- wav +::: + +### `decodeAudioData` + +Decodes audio data from either a file path or an ArrayBuffer. The optional `sampleRate` parameter lets you resample the decoded audio; +if not provided, the original sample rate from the file is used. + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterTypeDescription
inputArrayBufferArrayBuffer with audio data.
stringPath to audio file located on the device.
sampleRatenumberTarget sample rate for the decoded audio.
+ +#### Returns `Promise`. + +
+Example decoding with memory block +```tsx +const url = ... // url to an audio + + const buffer = await fetch(url) + .then((response) => response.arrayBuffer()) + // resample decoded audio to 48000 Hz + .then((arrayBuffer) => decodeAudioData(arrayBuffer, 48000)) + .catch((error) => { + console.error('Error decoding audio data source:', error); + return null; + }); +``` +
+ +
+Example using expo-asset library +```tsx +import { Asset } from 'expo-asset'; + +const buffer = await Asset.fromModule(require('@/assets/music/example.mp3')) + .downloadAsync() + .then((asset) => { + if (!asset.localUri) { + throw new Error('Failed to load audio asset'); + } + // sampleRate not provided, so file will be decoded in original sampleRate + return decodeAudioData(asset.localUri); + }) +``` +
+ +### `decodePCMInBase64` + +Decodes base64-encoded PCM audio data. + +#### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `base64String` | `string` | Base64-encoded PCM audio data. | +| `inputSampleRate` | `number` | Sample rate of the input PCM data. | +| `inputChannelCount` | `number` | Number of channels in the input PCM data. | +| `isInterleaved` | `boolean` | Whether the PCM data is interleaved. Default is `true`. | + +#### Returns `Promise` + + +
+Example decoding with data in base64 format +```tsx +const data = ... // data encoded in base64 string +// data is interleaved (Channel1, Channel2, Channel1, Channel2, ...) +const buffer = await decodeAudioData(data, 4800, 2, true); +``` +
diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioDecoder.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/AudioDecoder.cpp similarity index 52% rename from packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioDecoder.cpp rename to packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/AudioDecoder.cpp index b729f9ace..ac0e34a32 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioDecoder.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/AudioDecoder.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,28 +6,28 @@ #include #define MINIAUDIO_IMPLEMENTATION +#include +#include #include #ifndef AUDIO_API_TEST_SUITE #include #include #endif -#include -#include namespace audioapi { // Decoding audio in fixed-size chunks because total frame count can't be // determined in advance. Note: ma_decoder_get_length_in_pcm_frames() always // returns 0 for Vorbis decoders. -std::vector AudioDecoder::readAllPcmFrames( +std::vector AudioDecoder::readAllPcmFrames( ma_decoder &decoder, - int numChannels, - ma_uint64 &outFramesRead) { - std::vector buffer; - std::vector temp(CHUNK_SIZE * numChannels); - outFramesRead = 0; + int outputChannels) { + std::vector buffer; + std::vector temp(CHUNK_SIZE * outputChannels); + ma_uint64 outFramesRead = 0; +#ifndef AUDIO_API_TEST_SUITE while (true) { ma_uint64 tempFramesDecoded = 0; ma_decoder_read_pcm_frames( @@ -38,38 +39,46 @@ std::vector AudioDecoder::readAllPcmFrames( buffer.insert( buffer.end(), temp.data(), - temp.data() + tempFramesDecoded * numChannels); + temp.data() + tempFramesDecoded * outputChannels); outFramesRead += tempFramesDecoded; } + if (outFramesRead == 0) { + __android_log_print(ANDROID_LOG_ERROR, "AudioDecoder", "Failed to decode"); + } +#endif return buffer; } -std::shared_ptr AudioDecoder::makeAudioBusFromInt16Buffer( - const std::vector &buffer, - int numChannels, - float sampleRate) { - auto outputFrames = buffer.size() / numChannels; - auto audioBus = - std::make_shared(outputFrames, numChannels, sampleRate); +std::shared_ptr AudioDecoder::makeAudioBufferFromFloatBuffer( + const std::vector &buffer, + float outputSampleRate, + int outputChannels) { + if (buffer.empty()) { + return nullptr; + } - for (int ch = 0; ch < numChannels; ++ch) { + auto outputFrames = buffer.size() / outputChannels; + auto audioBus = std::make_shared( + outputFrames, outputChannels, outputSampleRate); + + for (int ch = 0; ch < outputChannels; ++ch) { auto channelData = audioBus->getChannel(ch)->getData(); for (int i = 0; i < outputFrames; ++i) { - channelData[i] = int16ToFloat(buffer[i * numChannels + ch]); + channelData[i] = buffer[i * outputChannels + ch]; } } - return audioBus; + return std::make_shared(audioBus); } -std::shared_ptr AudioDecoder::decodeWithFilePath( - const std::string &path) const { +std::shared_ptr AudioDecoder::decodeWithFilePath( + const std::string &path, + float sampleRate) { #ifndef AUDIO_API_TEST_SUITE - std::vector buffer; if (AudioDecoder::pathHasExtension(path, {".mp4", ".m4a", ".aac"})) { - buffer = ffmpegdecoding::decodeWithFilePath( - path, numChannels_, static_cast(sampleRate_)); - if (buffer.empty()) { + auto buffer = + ffmpegdecoder::decodeWithFilePath(path, static_cast(sampleRate)); + if (buffer == nullptr) { __android_log_print( ANDROID_LOG_ERROR, "AudioDecoder", @@ -77,11 +86,11 @@ std::shared_ptr AudioDecoder::decodeWithFilePath( path.c_str()); return nullptr; } - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); + return buffer; } ma_decoder decoder; - ma_decoder_config config = ma_decoder_config_init( - ma_format_s16, numChannels_, static_cast(sampleRate_)); + ma_decoder_config config = + ma_decoder_config_init(ma_format_f32, 0, static_cast(sampleRate)); ma_decoding_backend_vtable *customBackends[] = { ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; @@ -99,41 +108,38 @@ std::shared_ptr AudioDecoder::decodeWithFilePath( return nullptr; } - ma_uint64 framesRead = 0; - buffer = readAllPcmFrames(decoder, numChannels_, framesRead); - if (framesRead == 0) { - __android_log_print(ANDROID_LOG_ERROR, "AudioDecoder", "Failed to decode"); - ma_decoder_uninit(&decoder); - return nullptr; - } + auto outputSampleRate = static_cast(decoder.outputSampleRate); + auto outputChannels = static_cast(decoder.outputChannels); + std::vector buffer = readAllPcmFrames(decoder, outputChannels); ma_decoder_uninit(&decoder); - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); + return makeAudioBufferFromFloatBuffer( + buffer, outputSampleRate, outputChannels); #else return nullptr; #endif } -std::shared_ptr AudioDecoder::decodeWithMemoryBlock( +std::shared_ptr AudioDecoder::decodeWithMemoryBlock( const void *data, - size_t size) const { + size_t size, + float sampleRate) { #ifndef AUDIO_API_TEST_SUITE - std::vector buffer; const AudioFormat format = AudioDecoder::detectAudioFormat(data, size); if (format == AudioFormat::MP4 || format == AudioFormat::M4A || format == AudioFormat::AAC) { - buffer = ffmpegdecoding::decodeWithMemoryBlock( - data, size, numChannels_, sampleRate_); - if (buffer.empty()) { + auto buffer = ffmpegdecoder::decodeWithMemoryBlock( + data, size, static_cast(sampleRate)); + if (buffer == nullptr) { __android_log_print( ANDROID_LOG_ERROR, "AudioDecoder", "Failed to decode with FFmpeg"); return nullptr; } - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); + return buffer; } ma_decoder decoder; - ma_decoder_config config = ma_decoder_config_init( - ma_format_s16, numChannels_, static_cast(sampleRate_)); + ma_decoder_config config = + ma_decoder_config_init(ma_format_f32, 0, static_cast(sampleRate)); ma_decoding_backend_vtable *customBackends[] = { ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; @@ -151,50 +157,48 @@ std::shared_ptr AudioDecoder::decodeWithMemoryBlock( return nullptr; } - ma_uint64 framesRead = 0; - buffer = readAllPcmFrames(decoder, numChannels_, framesRead); - if (framesRead == 0) { - __android_log_print(ANDROID_LOG_ERROR, "AudioDecoder", "Failed to decode"); - ma_decoder_uninit(&decoder); - return nullptr; - } + auto outputSampleRate = static_cast(decoder.outputSampleRate); + auto outputChannels = static_cast(decoder.outputChannels); + std::vector buffer = readAllPcmFrames(decoder, outputChannels); ma_decoder_uninit(&decoder); - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); + return makeAudioBufferFromFloatBuffer( + buffer, outputSampleRate, outputChannels); #else return nullptr; #endif } -std::shared_ptr AudioDecoder::decodeWithPCMInBase64( +std::shared_ptr AudioDecoder::decodeWithPCMInBase64( const std::string &data, - float playbackSpeed) const { + float inputSampleRate, + int inputChannelCount, + bool interleaved) { auto decodedData = base64_decode(data, false); - const auto uint8Data = reinterpret_cast(decodedData.data()); - size_t framesDecoded = decodedData.size() / 2; - - std::vector buffer(framesDecoded); - for (size_t i = 0; i < framesDecoded; ++i) { - buffer[i] = - static_cast((uint8Data[i * 2 + 1] << 8) | uint8Data[i * 2]); - } + size_t numFramesDecoded = + decodedData.size() / (inputChannelCount * sizeof(int16_t)); - changePlaybackSpeedIfNeeded(buffer, framesDecoded, 1, playbackSpeed); - auto outputFrames = buffer.size(); + auto audioBus = std::make_shared( + numFramesDecoded, inputChannelCount, inputSampleRate); - auto audioBus = - std::make_shared(outputFrames, numChannels_, sampleRate_); - auto leftChannelData = audioBus->getChannel(0)->getData(); - auto rightChannelData = audioBus->getChannel(1)->getData(); + for (int ch = 0; ch < inputChannelCount; ++ch) { + auto channelData = audioBus->getChannel(ch)->getData(); - for (size_t i = 0; i < outputFrames; ++i) { - auto sample = int16ToFloat(buffer[i]); - leftChannelData[i] = sample; - rightChannelData[i] = sample; + for (size_t i = 0; i < numFramesDecoded; ++i) { + size_t offset; + if (interleaved) { + // Ch1, Ch2, Ch1, Ch2, ... + offset = (i * inputChannelCount + ch) * sizeof(int16_t); + } else { + // Ch1, Ch1, Ch1, ..., Ch2, Ch2, Ch2, ... + offset = (ch * numFramesDecoded + i) * sizeof(int16_t); + } + + channelData[i] = uint8ToFloat(uint8Data[offset], uint8Data[offset + 1]); + } } - - return audioBus; + return std::make_shared(audioBus); } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h index 8186018c8..c2bdfbb5c 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h @@ -1,15 +1,16 @@ #pragma once -#include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include -#include #include +#include #include @@ -22,29 +23,46 @@ using namespace facebook; class AudioAPIModuleInstaller { public: static void injectJSIBindings( - jsi::Runtime *jsiRuntime, - const std::shared_ptr &jsCallInvoker, - const std::shared_ptr &audioEventHandlerRegistry, - std::shared_ptr uiRuntime = nullptr) { - - auto createAudioContext = getCreateAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry, uiRuntime); - auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry); - auto createOfflineAudioContext = getCreateOfflineAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry, uiRuntime); - - jsiRuntime->global().setProperty(*jsiRuntime, "createAudioContext", createAudioContext); - jsiRuntime->global().setProperty(*jsiRuntime, "createAudioRecorder", createAudioRecorder); - jsiRuntime->global().setProperty(*jsiRuntime, "createOfflineAudioContext", createOfflineAudioContext); - - auto audioEventHandlerRegistryHostObject = std::make_shared(audioEventHandlerRegistry); - jsiRuntime->global().setProperty(*jsiRuntime, "AudioEventEmitter", jsi::Object::createFromHostObject(*jsiRuntime, audioEventHandlerRegistryHostObject)); + jsi::Runtime *jsiRuntime, + const std::shared_ptr &jsCallInvoker, + const std::shared_ptr + &audioEventHandlerRegistry, + std::shared_ptr uiRuntime = nullptr) { + auto createAudioContext = getCreateAudioContextFunction( + jsiRuntime, jsCallInvoker, audioEventHandlerRegistry, uiRuntime); + auto createAudioRecorder = + getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry); + auto createOfflineAudioContext = getCreateOfflineAudioContextFunction( + jsiRuntime, jsCallInvoker, audioEventHandlerRegistry, uiRuntime); + auto createAudioDecoder = + getCreateAudioDecoderFunction(jsiRuntime, jsCallInvoker); + + jsiRuntime->global().setProperty( + *jsiRuntime, "createAudioContext", createAudioContext); + jsiRuntime->global().setProperty( + *jsiRuntime, "createAudioRecorder", createAudioRecorder); + jsiRuntime->global().setProperty( + *jsiRuntime, "createOfflineAudioContext", createOfflineAudioContext); + jsiRuntime->global().setProperty( + *jsiRuntime, "createAudioDecoder", createAudioDecoder); + + auto audioEventHandlerRegistryHostObject = + std::make_shared( + audioEventHandlerRegistry); + jsiRuntime->global().setProperty( + *jsiRuntime, + "AudioEventEmitter", + jsi::Object::createFromHostObject( + *jsiRuntime, audioEventHandlerRegistryHostObject)); } private: static jsi::Function getCreateAudioContextFunction( - jsi::Runtime *jsiRuntime, - const std::shared_ptr &jsCallInvoker, - const std::shared_ptr &audioEventHandlerRegistry, - const std::weak_ptr &uiRuntime) { + jsi::Runtime *jsiRuntime, + const std::shared_ptr &jsCallInvoker, + const std::shared_ptr + &audioEventHandlerRegistry, + const std::weak_ptr &uiRuntime) { return jsi::Function::createFromHostFunction( *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "createAudioContext"), @@ -67,9 +85,14 @@ class AudioAPIModuleInstaller { auto runtimeRegistry = RuntimeRegistry{}; #endif - audioContext = std::make_shared(sampleRate, initSuspended, audioEventHandlerRegistry, runtimeRegistry); - auto audioContextHostObject = std::make_shared( - audioContext, &runtime, jsCallInvoker); + audioContext = std::make_shared( + sampleRate, + initSuspended, + audioEventHandlerRegistry, + runtimeRegistry); + auto audioContextHostObject = + std::make_shared( + audioContext, &runtime, jsCallInvoker); return jsi::Object::createFromHostObject( runtime, audioContextHostObject); @@ -77,10 +100,11 @@ class AudioAPIModuleInstaller { } static jsi::Function getCreateOfflineAudioContextFunction( - jsi::Runtime *jsiRuntime, - const std::shared_ptr &jsCallInvoker, - const std::shared_ptr &audioEventHandlerRegistry, - const std::weak_ptr &uiRuntime) { + jsi::Runtime *jsiRuntime, + const std::shared_ptr &jsCallInvoker, + const std::shared_ptr + &audioEventHandlerRegistry, + const std::weak_ptr &uiRuntime) { return jsi::Function::createFromHostFunction( *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "createOfflineAudioContext"), @@ -90,9 +114,9 @@ class AudioAPIModuleInstaller { const jsi::Value &thisValue, const jsi::Value *args, size_t count) -> jsi::Value { - auto numberOfChannels = static_cast(args[0].getNumber()); - auto length = static_cast(args[1].getNumber()); - auto sampleRate = static_cast(args[2].getNumber()); + auto numberOfChannels = static_cast(args[0].getNumber()); + auto length = static_cast(args[1].getNumber()); + auto sampleRate = static_cast(args[2].getNumber()); #if RN_AUDIO_API_ENABLE_WORKLETS auto runtimeRegistry = RuntimeRegistry{ @@ -103,9 +127,15 @@ class AudioAPIModuleInstaller { auto runtimeRegistry = RuntimeRegistry{}; #endif - auto offlineAudioContext = std::make_shared(numberOfChannels, length, sampleRate, audioEventHandlerRegistry, runtimeRegistry); - auto audioContextHostObject = std::make_shared( - offlineAudioContext, &runtime, jsCallInvoker); + auto offlineAudioContext = std::make_shared( + numberOfChannels, + length, + sampleRate, + audioEventHandlerRegistry, + runtimeRegistry); + auto audioContextHostObject = + std::make_shared( + offlineAudioContext, &runtime, jsCallInvoker); return jsi::Object::createFromHostObject( runtime, audioContextHostObject); @@ -113,8 +143,9 @@ class AudioAPIModuleInstaller { } static jsi::Function getCreateAudioRecorderFunction( - jsi::Runtime *jsiRuntime, - const std::shared_ptr &audioEventHandlerRegistry) { + jsi::Runtime *jsiRuntime, + const std::shared_ptr + &audioEventHandlerRegistry) { return jsi::Function::createFromHostFunction( *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "createAudioRecorder"), @@ -126,12 +157,37 @@ class AudioAPIModuleInstaller { size_t count) -> jsi::Value { auto options = args[0].getObject(runtime); - auto sampleRate = static_cast(options.getProperty(runtime, "sampleRate").getNumber()); - auto bufferLength = static_cast(options.getProperty(runtime, "bufferLengthInSamples").getNumber()); + auto sampleRate = static_cast( + options.getProperty(runtime, "sampleRate").getNumber()); + auto bufferLength = static_cast( + options.getProperty(runtime, "bufferLengthInSamples") + .getNumber()); - auto audioRecorderHostObject = std::make_shared(audioEventHandlerRegistry, sampleRate, bufferLength); + auto audioRecorderHostObject = + std::make_shared( + audioEventHandlerRegistry, sampleRate, bufferLength); - return jsi::Object::createFromHostObject(runtime, audioRecorderHostObject); + return jsi::Object::createFromHostObject( + runtime, audioRecorderHostObject); + }); + } + + static jsi::Function getCreateAudioDecoderFunction( + jsi::Runtime *jsiRuntime, + const std::shared_ptr &jsCallInvoker) { + return jsi::Function::createFromHostFunction( + *jsiRuntime, + jsi::PropNameID::forAscii(*jsiRuntime, "createAudioDecoder"), + 0, + [jsCallInvoker]( + jsi::Runtime &runtime, + const jsi::Value &thisValue, + const jsi::Value *args, + size_t count) -> jsi::Value { + auto audioDecoderHostObject = + std::make_shared(&runtime, jsCallInvoker); + return jsi::Object::createFromHostObject( + runtime, audioDecoderHostObject); }); } }; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.cpp b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.cpp index be60021cc..e08fe834d 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.cpp @@ -49,11 +49,7 @@ BaseAudioContextHostObject::BaseAudioContextHostObject( JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createBufferQueueSource), JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createBuffer), JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createPeriodicWave), - JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createAnalyser), - JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, decodeAudioData), - JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, decodeAudioDataSource), - JSI_EXPORT_FUNCTION( - BaseAudioContextHostObject, decodePCMAudioDataInBase64)); + JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createAnalyser)); } JSI_PROPERTY_GETTER_IMPL(BaseAudioContextHostObject, destination) { @@ -266,100 +262,4 @@ JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, createAnalyser) { auto analyserHostObject = std::make_shared(analyser); return jsi::Object::createFromHostObject(runtime, analyserHostObject); } - -JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, decodeAudioDataSource) { - auto sourcePath = args[0].getString(runtime).utf8(runtime); - - auto promise = promiseVendor_->createPromise( - [this, sourcePath](std::shared_ptr promise) { - std::thread([this, sourcePath, promise = std::move(promise)]() { - auto results = context_->decodeAudioDataSource(sourcePath); - - if (!results) { - promise->reject("Failed to decode audio data source."); - return; - } - - auto audioBufferHostObject = - std::make_shared(results); - - promise->resolve([audioBufferHostObject = std::move( - audioBufferHostObject)](jsi::Runtime &runtime) { - auto jsiObject = jsi::Object::createFromHostObject( - runtime, audioBufferHostObject); - jsiObject.setExternalMemoryPressure( - runtime, audioBufferHostObject->getSizeInBytes()); - return jsiObject; - }); - }).detach(); - }); - - return promise; -} - -JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, decodeAudioData) { - auto arrayBuffer = args[0] - .getObject(runtime) - .getPropertyAsObject(runtime, "buffer") - .getArrayBuffer(runtime); - auto data = arrayBuffer.data(runtime); - auto size = static_cast(arrayBuffer.size(runtime)); - - auto promise = promiseVendor_->createPromise( - [this, data, size](std::shared_ptr promise) { - std::thread([this, data, size, promise = std::move(promise)]() { - auto results = context_->decodeAudioData(data, size); - - if (!results) { - promise->reject("Failed to decode audio data source."); - return; - } - - auto audioBufferHostObject = - std::make_shared(results); - - promise->resolve([audioBufferHostObject = std::move( - audioBufferHostObject)](jsi::Runtime &runtime) { - auto jsiObject = jsi::Object::createFromHostObject( - runtime, audioBufferHostObject); - jsiObject.setExternalMemoryPressure( - runtime, audioBufferHostObject->getSizeInBytes()); - return jsiObject; - }); - }).detach(); - }); - - return promise; -} - -JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, decodePCMAudioDataInBase64) { - auto b64 = args[0].getString(runtime).utf8(runtime); - auto playbackSpeed = static_cast(args[1].getNumber()); - - auto promise = promiseVendor_->createPromise( - [this, b64, playbackSpeed](std::shared_ptr promise) { - std::thread([this, b64, playbackSpeed, promise = std::move(promise)]() { - auto results = context_->decodeWithPCMInBase64(b64, playbackSpeed); - - if (!results) { - promise->reject("Failed to decode audio data source."); - return; - } - - auto audioBufferHostObject = - std::make_shared(results); - - promise->resolve([audioBufferHostObject = std::move( - audioBufferHostObject)](jsi::Runtime &runtime) { - auto jsiObject = jsi::Object::createFromHostObject( - runtime, audioBufferHostObject); - jsiObject.setExternalMemoryPressure( - runtime, audioBufferHostObject->getSizeInBytes()); - return jsiObject; - }); - }).detach(); - }); - - return promise; -} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.h index ead53d32a..cc9694d7e 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.h @@ -41,9 +41,6 @@ class BaseAudioContextHostObject : public JsiHostObject { JSI_HOST_FUNCTION_DECL(createBuffer); JSI_HOST_FUNCTION_DECL(createPeriodicWave); JSI_HOST_FUNCTION_DECL(createAnalyser); - JSI_HOST_FUNCTION_DECL(decodeAudioDataSource); - JSI_HOST_FUNCTION_DECL(decodeAudioData); - JSI_HOST_FUNCTION_DECL(decodePCMAudioDataInBase64); std::shared_ptr context_; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.cpp b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.cpp new file mode 100644 index 000000000..317ce3ebe --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.cpp @@ -0,0 +1,107 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace audioapi { +AudioDecoderHostObject::AudioDecoderHostObject( + jsi::Runtime *runtime, + const std::shared_ptr &callInvoker) { + promiseVendor_ = std::make_shared(runtime, callInvoker); + addFunctions( + JSI_EXPORT_FUNCTION(AudioDecoderHostObject, decodeWithPCMInBase64), + JSI_EXPORT_FUNCTION(AudioDecoderHostObject, decodeWithFilePath), + JSI_EXPORT_FUNCTION(AudioDecoderHostObject, decodeWithMemoryBlock)); +} + +JSI_HOST_FUNCTION_IMPL(AudioDecoderHostObject, decodeWithMemoryBlock) { + auto arrayBuffer = args[0] + .getObject(runtime) + .getPropertyAsObject(runtime, "buffer") + .getArrayBuffer(runtime); + auto data = arrayBuffer.data(runtime); + auto size = static_cast(arrayBuffer.size(runtime)); + + auto sampleRate = args[1].getNumber(); + + return promiseVendor_->createAsyncPromise( + [data, size, sampleRate]( + jsi::Runtime &runtime) -> std::variant { + auto result = + AudioDecoder::decodeWithMemoryBlock(data, size, sampleRate); + + if (!result) { + return std::string("Failed to decode audio data."); + } + + auto audioBufferHostObject = + std::make_shared(result); + + auto jsiObject = + jsi::Object::createFromHostObject(runtime, audioBufferHostObject); + jsiObject.setExternalMemoryPressure( + runtime, audioBufferHostObject->getSizeInBytes()); + return jsiObject; + }); +} + +JSI_HOST_FUNCTION_IMPL(AudioDecoderHostObject, decodeWithFilePath) { + auto sourcePath = args[0].getString(runtime).utf8(runtime); + auto sampleRate = args[1].getNumber(); + + return promiseVendor_->createAsyncPromise( + [sourcePath, sampleRate]( + jsi::Runtime &runtime) -> std::variant { + auto result = AudioDecoder::decodeWithFilePath(sourcePath, sampleRate); + + if (!result) { + return std::string("Failed to decode audio data source."); + } + + auto audioBufferHostObject = + std::make_shared(result); + + auto jsiObject = + jsi::Object::createFromHostObject(runtime, audioBufferHostObject); + jsiObject.setExternalMemoryPressure( + runtime, audioBufferHostObject->getSizeInBytes()); + return jsiObject; + }); +} + +JSI_HOST_FUNCTION_IMPL(AudioDecoderHostObject, decodeWithPCMInBase64) { + auto b64 = args[0].getString(runtime).utf8(runtime); + auto inputSampleRate = args[1].getNumber(); + auto inputChannelCount = args[2].getNumber(); + auto interleaved = args[3].getBool(); + + return promiseVendor_->createAsyncPromise( + [b64, inputSampleRate, inputChannelCount, interleaved]( + jsi::Runtime &runtime) -> std::variant { + auto result = AudioDecoder::decodeWithPCMInBase64( + b64, inputSampleRate, inputChannelCount, interleaved); + + if (!result) { + return std::string("Failed to decode audio data source."); + } + + auto audioBufferHostObject = + std::make_shared(result); + + auto jsiObject = + jsi::Object::createFromHostObject(runtime, audioBufferHostObject); + jsiObject.setExternalMemoryPressure( + runtime, audioBufferHostObject->getSizeInBytes()); + return jsiObject; + }); +} + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.h new file mode 100644 index 000000000..57d25c17f --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/utils/AudioDecoderHostObject.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace audioapi { +using namespace facebook; + +class AudioDecoderHostObject : public JsiHostObject { + public: + explicit AudioDecoderHostObject( + jsi::Runtime *runtime, + const std::shared_ptr &callInvoker); + JSI_HOST_FUNCTION_DECL(decodeWithMemoryBlock); + JSI_HOST_FUNCTION_DECL(decodeWithFilePath); + JSI_HOST_FUNCTION_DECL(decodeWithPCMInBase64); + + private: + std::shared_ptr promiseVendor_; +}; +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp index 019481140..a0992a2e8 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp @@ -6,7 +6,6 @@ #include #include -#include #include namespace audioapi { @@ -26,7 +25,6 @@ AudioContext::AudioContext( #endif sampleRate_ = sampleRate; - audioDecoder_ = std::make_shared(sampleRate); if (initSuspended) { playerHasBeenStarted_ = false; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.cpp index ca89dfafc..7a63739cc 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.cpp @@ -176,41 +176,6 @@ std::shared_ptr BaseAudioContext::createAnalyser() { return analyser; } -std::shared_ptr BaseAudioContext::decodeAudioDataSource( - const std::string &path) { - auto audioBus = audioDecoder_->decodeWithFilePath(path); - - if (!audioBus) { - return nullptr; - } - - return std::make_shared(audioBus); -} - -std::shared_ptr BaseAudioContext::decodeAudioData( - const void *data, - size_t size) { - auto audioBus = audioDecoder_->decodeWithMemoryBlock(data, size); - - if (!audioBus) { - return nullptr; - } - - return std::make_shared(audioBus); -} - -std::shared_ptr BaseAudioContext::decodeWithPCMInBase64( - const std::string &data, - float playbackSpeed) { - auto audioBus = audioDecoder_->decodeWithPCMInBase64(data, playbackSpeed); - - if (!audioBus) { - return nullptr; - } - - return std::make_shared(audioBus); -} - AudioNodeManager *BaseAudioContext::getNodeManager() { return nodeManager_.get(); } diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.h index 92e23b3bf..69ae155be 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/BaseAudioContext.h @@ -3,15 +3,14 @@ #include #include #include - +#include +#include +#include #include #include #include #include #include -#include -#include -#include namespace audioapi { @@ -27,7 +26,6 @@ class BiquadFilterNode; class AudioDestinationNode; class AudioBufferSourceNode; class AudioBufferQueueSourceNode; -class AudioDecoder; class AnalyserNode; class AudioEventHandlerRegistry; class IAudioEventHandlerRegistry; @@ -68,10 +66,6 @@ class BaseAudioContext { int length); std::shared_ptr createAnalyser(); - std::shared_ptr decodeAudioDataSource(const std::string &path); - std::shared_ptr decodeAudioData(const void *data, size_t size); - std::shared_ptr decodeWithPCMInBase64(const std::string &data, float playbackSpeed); - std::shared_ptr getBasicWaveForm(OscillatorType type); [[nodiscard]] float getNyquistFrequency() const; AudioNodeManager *getNodeManager(); @@ -85,9 +79,7 @@ class BaseAudioContext { std::shared_ptr destination_; // init in AudioContext or OfflineContext constructor - std::shared_ptr audioDecoder_ {}; - // init in AudioContext or OfflineContext constructor - float sampleRate_ {}; + float sampleRate_{}; ContextState state_ = ContextState::RUNNING; std::shared_ptr nodeManager_; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/OfflineAudioContext.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/OfflineAudioContext.cpp index 975d900d5..bffb0dcff 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/OfflineAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/OfflineAudioContext.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,6 @@ OfflineAudioContext::OfflineAudioContext( numberOfChannels_(numberOfChannels), currentSampleFrame_(0) { sampleRate_ = sampleRate; - audioDecoder_ = std::make_shared(sampleRate_); resultBus_ = std::make_shared( static_cast(length_), numberOfChannels_, sampleRate_); } diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioFormat.h b/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioFormat.h new file mode 100644 index 000000000..509392cce --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioFormat.h @@ -0,0 +1,16 @@ +#pragma once + +namespace audioapi { + +enum class AudioFormat { + UNKNOWN, + WAV, + OGG, + FLAC, + AAC, + MP3, + M4A, + MP4, + MOV +}; +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioDecoder.h b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioDecoder.h index ca0b81bb1..c1e0a7283 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioDecoder.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/AudioDecoder.h @@ -1,124 +1,68 @@ #pragma once +#include #include #include +#include +#include #include #include #include -#include -#include namespace audioapi { -enum class AudioFormat { - UNKNOWN, - WAV, - OGG, - FLAC, - AAC, - MP3, - M4A, - MP4, - MOV -}; - -class AudioBus; +class AudioBuffer; static constexpr int CHUNK_SIZE = 4096; class AudioDecoder { public: - explicit AudioDecoder(float sampleRate) : sampleRate_(sampleRate) {} + AudioDecoder() = delete; - [[nodiscard]] std::shared_ptr decodeWithFilePath( - const std::string &path) const; - [[nodiscard]] std::shared_ptr decodeWithMemoryBlock( - const void *data, - size_t size) const; - [[nodiscard]] std::shared_ptr decodeWithPCMInBase64( - const std::string &data, - float playbackSpeed) const; + [[nodiscard]] static std::shared_ptr decodeWithFilePath(const std::string &path, float sampleRate); + [[nodiscard]] static std::shared_ptr + decodeWithMemoryBlock(const void *data, size_t size, float sampleRate); + [[nodiscard]] static std::shared_ptr + decodeWithPCMInBase64(const std::string &data, float inputSampleRate, int inputChannelCount, bool interleaved); private: - float sampleRate_; - int numChannels_ = 2; - - static std::vector readAllPcmFrames( - ma_decoder &decoder, - int numChannels, - ma_uint64 &outFramesRead); - static std::shared_ptr makeAudioBusFromInt16Buffer( - const std::vector &buffer, - int numChannels, - float sampleRate); - - void changePlaybackSpeedIfNeeded( - std::vector &buffer, - size_t framesDecoded, - int numChannels, - float playbackSpeed) const { - if (playbackSpeed == 1.0f) { - return; - } - - auto stretcher = stretch_init( - static_cast(sampleRate_ / 333.0f), - static_cast(sampleRate_ / 55.0f), - numChannels, - 0x1); + static std::vector readAllPcmFrames(ma_decoder &decoder, int outputChannels); + static std::shared_ptr + makeAudioBufferFromFloatBuffer(const std::vector &buffer, float outputSampleRate, int outputChannels); - int maxOutputFrames = stretch_output_capacity( - stretcher, static_cast(framesDecoded), 1 / playbackSpeed); - std::vector stretchedBuffer(maxOutputFrames); - - int outputFrames = stretch_samples( - stretcher, - buffer.data(), - static_cast(framesDecoded), - stretchedBuffer.data(), - 1 / playbackSpeed); - - outputFrames += - stretch_flush(stretcher, stretchedBuffer.data() + (outputFrames)); - stretchedBuffer.resize(outputFrames); - - buffer = stretchedBuffer; - - stretch_deinit(stretcher); - } - - static AudioFormat detectAudioFormat(const void* data, size_t size) { - if (size < 12) return AudioFormat::UNKNOWN; - const auto* bytes = static_cast(data); + static AudioFormat detectAudioFormat(const void *data, size_t size) { + if (size < 12) + return AudioFormat::UNKNOWN; + const auto *bytes = static_cast(data); // WAV/RIFF if (std::memcmp(bytes, "RIFF", 4) == 0 && std::memcmp(bytes + 8, "WAVE", 4) == 0) - return AudioFormat::WAV; + return AudioFormat::WAV; // OGG if (std::memcmp(bytes, "OggS", 4) == 0) - return AudioFormat::OGG; + return AudioFormat::OGG; // FLAC if (std::memcmp(bytes, "fLaC", 4) == 0) - return AudioFormat::FLAC; + return AudioFormat::FLAC; // AAC starts with 0xFF 0xF1 or 0xFF 0xF9 if (bytes[0] == 0xFF && (bytes[1] & 0xF6) == 0xF0) - return AudioFormat::AAC; + return AudioFormat::AAC; // MP3: "ID3" or 11-bit frame sync (0xFF 0xE0) if (std::memcmp(bytes, "ID3", 3) == 0) - return AudioFormat::MP3; + return AudioFormat::MP3; if (bytes[0] == 0xFF && (bytes[1] & 0xE0) == 0xE0) - return AudioFormat::MP3; + return AudioFormat::MP3; if (std::memcmp(bytes + 4, "ftyp", 4) == 0) { - if (std::memcmp(bytes + 8, "M4A ", 4) == 0) - return AudioFormat::M4A; - else if (std::memcmp(bytes + 8, "qt ", 4) == 0) - return AudioFormat::MOV; - return AudioFormat::MP4; + if (std::memcmp(bytes + 8, "M4A ", 4) == 0) + return AudioFormat::M4A; + else if (std::memcmp(bytes + 8, "qt ", 4) == 0) + return AudioFormat::MOV; + return AudioFormat::MP4; } return AudioFormat::UNKNOWN; } @@ -126,19 +70,21 @@ class AudioDecoder { static inline bool pathHasExtension(const std::string &path, const std::vector &extensions) { std::string pathLower = path; std::transform(pathLower.begin(), pathLower.end(), pathLower.begin(), ::tolower); - for (const auto& ext : extensions) { - if (pathLower.ends_with(ext)) - return true; + for (const auto &ext : extensions) { + if (pathLower.ends_with(ext)) + return true; } return false; } - [[nodiscard]] static inline int16_t floatToInt16(float sample) { - return static_cast(sample * 32768.0f); + return static_cast(sample * INT16_MAX); } [[nodiscard]] static inline float int16ToFloat(int16_t sample) { - return static_cast(sample) / 32768.0f; + return static_cast(sample) / INT16_MAX; + } + [[nodiscard]] static inline float uint8ToFloat(uint8_t byte1, uint8_t byte2) { + return static_cast(static_cast((byte2 << 8) | byte1)) / INT16_MAX; } }; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.cpp b/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.cpp index fcab67432..e235380f6 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.cpp @@ -1,16 +1,20 @@ /* - * This file dynamically links to the FFmpeg library, which is licensed under the - * GNU Lesser General Public License (LGPL) version 2.1 or later. + * This file dynamically links to the FFmpeg library, which is licensed under + * the GNU Lesser General Public License (LGPL) version 2.1 or later. * - * Our own code in this file is licensed under the MIT License and dynamic linking - * allows you to use this code without your entire project being subject to the - * terms of the LGPL. However, note that if you link statically to FFmpeg, you must - * comply with the terms of the LGPL for FFmpeg itself. + * Our own code in this file is licensed under the MIT License and dynamic + * linking allows you to use this code without your entire project being subject + * to the terms of the LGPL. However, note that if you link statically to + * FFmpeg, you must comply with the terms of the LGPL for FFmpeg itself. */ -#include "FFmpegDecoding.h" +#include +#include +#include +#include +#include -namespace audioapi::ffmpegdecoding { +namespace audioapi::ffmpegdecoder { int read_packet(void *opaque, uint8_t *buf, int buf_size) { MemoryIOContext *ctx = static_cast(opaque); @@ -51,42 +55,87 @@ int64_t seek_packet(void *opaque, int64_t offset, int whence) { return ctx->pos; } -std::vector readAllPcmFrames( +void convertFrameToBuffer( + SwrContext *swr, + AVFrame *frame, + int output_channel_count, + std::vector &buffer, + size_t &framesRead, + uint8_t **&resampled_data, + int &max_resampled_samples) { + const int out_samples = swr_get_out_samples(swr, frame->nb_samples); + if (out_samples > max_resampled_samples) { + av_freep(&resampled_data[0]); + av_freep(&resampled_data); + max_resampled_samples = out_samples; + + if (av_samples_alloc_array_and_samples( + &resampled_data, + nullptr, + output_channel_count, + max_resampled_samples, + AV_SAMPLE_FMT_FLT, + 0) < 0) { + return; + } + } + + int converted_samples = swr_convert( + swr, + resampled_data, + max_resampled_samples, + const_cast(frame->data), + frame->nb_samples); + + if (converted_samples > 0) { + const size_t current_size = buffer.size(); + const size_t new_samples = + static_cast(converted_samples) * output_channel_count; + buffer.resize(current_size + new_samples); + memcpy( + buffer.data() + current_size, + resampled_data[0], + new_samples * sizeof(float)); + framesRead += converted_samples; + } +} + +std::vector readAllPcmFrames( AVFormatContext *fmt_ctx, AVCodecContext *codec_ctx, int out_sample_rate, + int output_channel_count, int audio_stream_index, - int channels, size_t &framesRead) { - std::vector buffer; - SwrContext *swr_ctx = swr_alloc(); - if (swr_ctx == nullptr) { + framesRead = 0; + std::vector buffer; + auto swr = std::unique_ptr>( + swr_alloc(), [](SwrContext *ctx) { swr_free(&ctx); }); + + if (swr == nullptr) return buffer; - } - av_opt_set_chlayout(swr_ctx, "in_chlayout", &codec_ctx->ch_layout, 0); - av_opt_set_int(swr_ctx, "in_sample_rate", codec_ctx->sample_rate, 0); - av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", codec_ctx->sample_fmt, 0); + av_opt_set_chlayout(swr.get(), "in_chlayout", &codec_ctx->ch_layout, 0); + av_opt_set_int(swr.get(), "in_sample_rate", codec_ctx->sample_rate, 0); + av_opt_set_sample_fmt(swr.get(), "in_sample_fmt", codec_ctx->sample_fmt, 0); AVChannelLayout out_ch_layout; - av_channel_layout_default(&out_ch_layout, channels); - av_opt_set_chlayout(swr_ctx, "out_chlayout", &out_ch_layout, 0); - av_opt_set_int(swr_ctx, "out_sample_rate", out_sample_rate, 0); - av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); + av_channel_layout_default(&out_ch_layout, output_channel_count); + av_opt_set_chlayout(swr.get(), "out_chlayout", &out_ch_layout, 0); + av_opt_set_int(swr.get(), "out_sample_rate", out_sample_rate, 0); + av_opt_set_sample_fmt(swr.get(), "out_sample_fmt", AV_SAMPLE_FMT_FLT, 0); - if (swr_init(swr_ctx) < 0) { - swr_free(&swr_ctx); + if (swr_init(swr.get()) < 0) { av_channel_layout_uninit(&out_ch_layout); return buffer; } - AVPacket *packet = av_packet_alloc(); - AVFrame *frame = av_frame_alloc(); - + auto packet = std::unique_ptr>( + av_packet_alloc(), [](AVPacket *p) { av_packet_free(&p); }); + auto frame = std::unique_ptr>( + av_frame_alloc(), [](AVFrame *p) { av_frame_free(&p); }); + if (packet == nullptr || frame == nullptr) { - if (packet != nullptr) av_packet_free(&packet); - if (frame != nullptr) av_frame_free(&frame); - swr_free(&swr_ctx); av_channel_layout_uninit(&out_ch_layout); return buffer; } @@ -94,312 +143,222 @@ std::vector readAllPcmFrames( // Allocate buffer for resampled data uint8_t **resampled_data = nullptr; int max_resampled_samples = 4096; // Initial size - int ret = av_samples_alloc_array_and_samples( - &resampled_data, - nullptr, - channels, - max_resampled_samples, - AV_SAMPLE_FMT_S16, - 0); - - if (ret < 0) { - av_frame_free(&frame); - av_packet_free(&packet); - swr_free(&swr_ctx); + if (av_samples_alloc_array_and_samples( + &resampled_data, + nullptr, + output_channel_count, + max_resampled_samples, + AV_SAMPLE_FMT_FLT, + 0) < 0) { av_channel_layout_uninit(&out_ch_layout); return buffer; } - framesRead = 0; - - while (av_read_frame(fmt_ctx, packet) >= 0) { + while (av_read_frame(fmt_ctx, packet.get()) >= 0) { if (packet->stream_index == audio_stream_index) { - if (avcodec_send_packet(codec_ctx, packet) == 0) { - while (avcodec_receive_frame(codec_ctx, frame) == 0) { - // Check if we need more buffer space - int out_samples = swr_get_out_samples(swr_ctx, frame->nb_samples); - if (out_samples > max_resampled_samples) { - if (resampled_data != nullptr) { - av_freep(&resampled_data[0]); - av_freep(&resampled_data); - } - - max_resampled_samples = out_samples; - ret = av_samples_alloc_array_and_samples( - &resampled_data, - nullptr, - channels, - max_resampled_samples, - AV_SAMPLE_FMT_S16, - 0); - - if (ret < 0) { - break; // Exit on allocation failure - } - } - - int converted_samples = swr_convert( - swr_ctx, + if (avcodec_send_packet(codec_ctx, packet.get()) == 0) { + while (avcodec_receive_frame(codec_ctx, frame.get()) == 0) { + convertFrameToBuffer( + swr.get(), + frame.get(), + output_channel_count, + buffer, + framesRead, resampled_data, - max_resampled_samples, - (const uint8_t **)frame->data, - frame->nb_samples); - - if (converted_samples > 0) { - size_t current_size = buffer.size(); - size_t new_samples = converted_samples * channels; - buffer.resize(current_size + new_samples); - memcpy( - buffer.data() + current_size, - resampled_data[0], - new_samples * sizeof(int16_t)); - - framesRead += converted_samples; - } + max_resampled_samples); } } } - av_packet_unref(packet); + av_packet_unref(packet.get()); } // Flush decoder avcodec_send_packet(codec_ctx, nullptr); - while (avcodec_receive_frame(codec_ctx, frame) == 0) { - int out_samples = swr_get_out_samples(swr_ctx, frame->nb_samples); - if (out_samples > max_resampled_samples) { - if (resampled_data != nullptr) { - av_freep(&resampled_data[0]); - av_freep(&resampled_data); - } - - max_resampled_samples = out_samples; - ret = av_samples_alloc_array_and_samples( - &resampled_data, - nullptr, - channels, - max_resampled_samples, - AV_SAMPLE_FMT_S16, - 0); - - if (ret < 0) { - break; - } - } - - int converted_samples = swr_convert( - swr_ctx, + while (avcodec_receive_frame(codec_ctx, frame.get()) == 0) { + convertFrameToBuffer( + swr.get(), + frame.get(), + output_channel_count, + buffer, + framesRead, resampled_data, - max_resampled_samples, - (const uint8_t **)frame->data, - frame->nb_samples); - - if (converted_samples > 0) { - size_t current_size = buffer.size(); - size_t new_samples = converted_samples * channels; - buffer.resize(current_size + new_samples); - memcpy( - buffer.data() + current_size, - resampled_data[0], - new_samples * sizeof(int16_t)); - - framesRead += converted_samples; - } + max_resampled_samples); } - if (resampled_data != nullptr) { - av_freep(&resampled_data[0]); - av_freep(&resampled_data); - } - swr_free(&swr_ctx); + av_freep(&resampled_data[0]); + av_freep(&resampled_data); av_channel_layout_uninit(&out_ch_layout); - av_frame_free(&frame); - av_packet_free(&packet); return buffer; } -std::vector decodeWithMemoryBlock(const void *data, size_t size, const int channel_count, int sample_rate) { - if (data == nullptr || size == 0) { - return {}; - } - - MemoryIOContext io_ctx; - io_ctx.data = static_cast(data); - io_ctx.size = size; - io_ctx.pos = 0; - - constexpr size_t buffer_size = 4096; - uint8_t *io_buffer = static_cast(av_malloc(buffer_size)); - if (io_buffer == nullptr) { - return {}; +inline int findAudioStreamIndex(AVFormatContext *fmt_ctx) { + for (int i = 0; i < fmt_ctx->nb_streams; i++) { + if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + return i; } + } + return -1; +} - AVIOContext *avio_ctx = avio_alloc_context( - io_buffer, buffer_size, 0, &io_ctx, read_packet, nullptr, seek_packet); - - if (avio_ctx == nullptr) { - av_free(io_buffer); - return {}; - } +bool setupDecoderContext( + AVFormatContext *fmt_ctx, + int &audio_stream_index, + std::unique_ptr> + &codec_ctx) { + audio_stream_index = findAudioStreamIndex(fmt_ctx); + if (audio_stream_index == -1) { + return false; + } - // Create format context and set custom IO - AVFormatContext *fmt_ctx = avformat_alloc_context(); - if (fmt_ctx == nullptr) { - avio_context_free(&avio_ctx); - return {}; - } - - fmt_ctx->pb = avio_ctx; - - // Open input from memory - if (avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr) < 0) { - avformat_free_context(fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + AVCodecParameters *codecpar = fmt_ctx->streams[audio_stream_index]->codecpar; + const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); + if (codec == nullptr) { + return false; + } - // Find stream info - if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + AVCodecContext *raw_codec_ctx = avcodec_alloc_context3(codec); + if (raw_codec_ctx == nullptr) { + return false; + } - int audio_stream_index = -1; - for (int i = 0; i < fmt_ctx->nb_streams; i++) { - if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - audio_stream_index = i; - break; - } - } + codec_ctx.reset(raw_codec_ctx); + if (avcodec_parameters_to_context(codec_ctx.get(), codecpar) < 0) { + return false; + } + if (avcodec_open2(codec_ctx.get(), codec, nullptr) < 0) { + return false; + } - if (audio_stream_index == -1) { - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + return true; +} - AVCodecParameters *codecpar = fmt_ctx->streams[audio_stream_index]->codecpar; +std::shared_ptr decodeAudioFrames( + AVFormatContext *fmt_ctx, + AVCodecContext *codec_ctx, + int audio_stream_index, + int sample_rate) { + size_t framesRead = 0; + int output_sample_rate = + (sample_rate > 0) ? sample_rate : codec_ctx->sample_rate; + int output_channel_count = codec_ctx->ch_layout.nb_channels; + + std::vector decoded_buffer = readAllPcmFrames( + fmt_ctx, + codec_ctx, + output_sample_rate, + output_channel_count, + audio_stream_index, + framesRead); - // Find decoder - const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); - if (codec == nullptr) { - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + if (framesRead == 0 || decoded_buffer.empty()) { + return nullptr; + } - // Allocate and setup codec context - AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); - if (codec_ctx == nullptr) { - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + auto outputFrames = decoded_buffer.size() / output_channel_count; + auto audioBus = std::make_shared( + outputFrames, output_channel_count, output_sample_rate); - if (avcodec_parameters_to_context(codec_ctx, codecpar) < 0) { - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; + for (int ch = 0; ch < output_channel_count; ++ch) { + auto channelData = audioBus->getChannel(ch)->getData(); + for (int i = 0; i < outputFrames; ++i) { + channelData[i] = decoded_buffer[i * output_channel_count + ch]; } + } + return std::make_shared(audioBus); +} - if (avcodec_open2(codec_ctx, codec, nullptr) < 0) { - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } +std::shared_ptr +decodeWithMemoryBlock(const void *data, size_t size, int sample_rate) { + if (data == nullptr || size == 0) { + return nullptr; + } - // Get actual channel count from the decoded stream - int actual_channels = codec_ctx->ch_layout.nb_channels; + MemoryIOContext io_ctx{static_cast(data), size, 0}; - // Validate channel count - if (actual_channels <= 0 || actual_channels > 8) { - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); - return {}; - } + constexpr size_t buffer_size = 4096; + auto io_buffer = std::unique_ptr( + static_cast(av_malloc(buffer_size)), &av_free); + if (io_buffer == nullptr) { + return nullptr; + } - // Decode all frames - size_t framesRead = 0; - std::vector decoded_buffer = readAllPcmFrames( - fmt_ctx, codec_ctx, sample_rate, audio_stream_index, channel_count, framesRead); + auto avio_ctx = + std::unique_ptr>( + avio_alloc_context( + io_buffer.get(), + buffer_size, + 0, + &io_ctx, + read_packet, + nullptr, + seek_packet), + [](AVIOContext *ctx) { avio_context_free(&ctx); }); + if (avio_ctx == nullptr) { + return nullptr; + } - // Cleanup - Note: avio_context_free will free the io_buffer - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - avio_context_free(&avio_ctx); + AVFormatContext *raw_fmt_ctx = avformat_alloc_context(); + if (raw_fmt_ctx == nullptr) { + return nullptr; + } + raw_fmt_ctx->pb = avio_ctx.get(); - if (framesRead == 0 || decoded_buffer.empty()) { - return {}; - } + if (avformat_open_input(&raw_fmt_ctx, nullptr, nullptr, nullptr) < 0) { + avformat_free_context(raw_fmt_ctx); + return nullptr; + } - return decoded_buffer; -} + auto fmt_ctx = + std::unique_ptr( + raw_fmt_ctx, &avformat_free_context); -std::vector decodeWithFilePath(const std::string &path, const int channel_count, int sample_rate) { - if (path.empty()) { - return {}; + if (avformat_find_stream_info(fmt_ctx.get(), nullptr) < 0) { + return nullptr; } - AVFormatContext *fmt_ctx = nullptr; - if (avformat_open_input(&fmt_ctx, path.c_str(), nullptr, nullptr) < 0) { - return {}; - } - if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { - avformat_close_input(&fmt_ctx); - return {}; - } + auto codec_ctx = + std::unique_ptr>( + nullptr, [](AVCodecContext *ctx) { avcodec_free_context(&ctx); }); int audio_stream_index = -1; - for (int i = 0; i < fmt_ctx->nb_streams; i++) { - if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - audio_stream_index = i; - break; - } - } - if (audio_stream_index == -1) { - avformat_close_input(&fmt_ctx); - return {}; + if (!setupDecoderContext(fmt_ctx.get(), audio_stream_index, codec_ctx)) { + return nullptr; } - AVCodecParameters *codecpar = fmt_ctx->streams[audio_stream_index]->codecpar; - const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); - if (codec == nullptr) { - avformat_close_input(&fmt_ctx); - return {}; - } - AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); - if (codec_ctx == nullptr) { - avformat_close_input(&fmt_ctx); - return {}; - } + return decodeAudioFrames( + fmt_ctx.get(), codec_ctx.get(), audio_stream_index, sample_rate); +} - if (avcodec_parameters_to_context(codec_ctx, codecpar) < 0) { - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - return {}; +std::shared_ptr decodeWithFilePath( + const std::string &path, + int sample_rate) { + if (path.empty()) { + return nullptr; } - if (avcodec_open2(codec_ctx, codec, nullptr) < 0) { - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); - return {}; - } + AVFormatContext *raw_fmt_ctx = nullptr; + if (avformat_open_input(&raw_fmt_ctx, path.c_str(), nullptr, nullptr) < 0) + return nullptr; - size_t framesRead = 0; - std::vector decoded_buffer = readAllPcmFrames( - fmt_ctx, codec_ctx, sample_rate, audio_stream_index, channel_count, framesRead); + auto fmt_ctx = + std::unique_ptr>( + raw_fmt_ctx, + [](AVFormatContext *ctx) { avformat_close_input(&ctx); }); - avcodec_free_context(&codec_ctx); - avformat_close_input(&fmt_ctx); + if (avformat_find_stream_info(fmt_ctx.get(), nullptr) < 0) { + return nullptr; + } - if (framesRead == 0 || decoded_buffer.empty()) { - return {}; + auto codec_ctx = + std::unique_ptr>( + nullptr, [](AVCodecContext *ctx) { avcodec_free_context(&ctx); }); + int audio_stream_index = -1; + if (!setupDecoderContext(fmt_ctx.get(), audio_stream_index, codec_ctx)) { + return nullptr; } - return decoded_buffer; + return decodeAudioFrames( + fmt_ctx.get(), codec_ctx.get(), audio_stream_index, sample_rate); } } // namespace audioapi::ffmpegdecoder diff --git a/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.h b/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.h index 0631154ba..f00047d7f 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/libs/ffmpeg/FFmpegDecoding.h @@ -1,11 +1,11 @@ /* - * This file dynamically links to the FFmpeg library, which is licensed under the - * GNU Lesser General Public License (LGPL) version 2.1 or later. + * This file dynamically links to the FFmpeg library, which is licensed under + * the GNU Lesser General Public License (LGPL) version 2.1 or later. * - * Our own code in this file is licensed under the MIT License and dynamic linking - * allows you to use this code without your entire project being subject to the - * terms of the LGPL. However, note that if you link statically to FFmpeg, you must - * comply with the terms of the LGPL for FFmpeg itself. + * Our own code in this file is licensed under the MIT License and dynamic + * linking allows you to use this code without your entire project being subject + * to the terms of the LGPL. However, note that if you link statically to + * FFmpeg, you must comply with the terms of the LGPL for FFmpeg itself. */ #include @@ -14,24 +14,62 @@ #include extern "C" { - #include - #include - #include - #include +#include +#include +#include +#include } +class AudioBuffer; -namespace audioapi::ffmpegdecoding { +namespace audioapi::ffmpegdecoder { // Custom IO context for reading from memory struct MemoryIOContext { - const uint8_t *data; - size_t size; - size_t pos; - }; + const uint8_t *data; + size_t size; + size_t pos; +}; + +struct AudioStreamContext { + AVFormatContext *fmt_ctx = nullptr; + AVCodecContext *codec_ctx = nullptr; + int audio_stream_index = -1; +}; int read_packet(void *opaque, uint8_t *buf, int buf_size); int64_t seek_packet(void *opaque, int64_t offset, int whence); -std::vector readAllPcmFrames(AVFormatContext *fmt_ctx, AVCodecContext *codec_ctx, int out_sample_rate, int audio_stream_index, int channels, size_t &framesRead); -std::vector decodeWithMemoryBlock(const void *data, size_t size, const int channel_count, int sample_rate); -std::vector decodeWithFilePath(const std::string &path, const int channel_count, int sample_rate); +inline int findAudioStreamIndex(AVFormatContext *fmt_ctx); +std::vector readAllPcmFrames( + AVFormatContext *fmt_ctx, + AVCodecContext *codec_ctx, + int out_sample_rate, + int output_channel_count, + int audio_stream_index, + size_t &framesRead); + +void convertFrameToBuffer( + SwrContext *swr, + AVFrame *frame, + int output_channel_count, + std::vector &buffer, + size_t &framesRead, + uint8_t **&resampled_data, + int &max_resampled_samples); +bool setupDecoderContext( + AVFormatContext *fmt_ctx, + int &audio_stream_index, + std::unique_ptr + &codec_ctx); +std::shared_ptr decodeAudioFrames( + AVFormatContext *fmt_ctx, + AVCodecContext *codec_ctx, + int audio_stream_index, + int sample_rate); + +std::shared_ptr +decodeWithMemoryBlock(const void *data, size_t size, int sample_rate); + +std::shared_ptr decodeWithFilePath( + const std::string &path, + int sample_rate); -} // namespace audioapi::ffmpegdecoder \ No newline at end of file +} // namespace audioapi::ffmpegdecoder diff --git a/packages/react-native-audio-api/common/cpp/test/CMakeLists.txt b/packages/react-native-audio-api/common/cpp/test/CMakeLists.txt index d1eafd97e..2126a7b23 100644 --- a/packages/react-native-audio-api/common/cpp/test/CMakeLists.txt +++ b/packages/react-native-audio-api/common/cpp/test/CMakeLists.txt @@ -21,7 +21,7 @@ enable_testing() file(GLOB_RECURSE RNAUDIOAPI_SRC CONFIGURE_DEPENDS "${ROOT}/node_modules/react-native-audio-api/common/cpp/audioapi/*.cpp" - "${ROOT}/node_modules/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioDecoder.cpp" + "${ROOT}/node_modules/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/AudioDecoder.cpp" ) # exclude HostObjects from tests diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/AudioDecoder.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/AudioDecoder.mm deleted file mode 100644 index 93ecf407f..000000000 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/AudioDecoder.mm +++ /dev/null @@ -1,156 +0,0 @@ -#define MINIAUDIO_IMPLEMENTATION -#import - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace audioapi { - -// Decoding audio in fixed-size chunks because total frame count can't be -// determined in advance. Note: ma_decoder_get_length_in_pcm_frames() always -// returns 0 for Vorbis decoders. -std::vector AudioDecoder::readAllPcmFrames(ma_decoder &decoder, int numChannels, ma_uint64 &outFramesRead) -{ - std::vector buffer; - std::vector temp(CHUNK_SIZE * numChannels); - outFramesRead = 0; - - while (true) { - ma_uint64 tempFramesDecoded = 0; - ma_decoder_read_pcm_frames(&decoder, temp.data(), CHUNK_SIZE, &tempFramesDecoded); - if (tempFramesDecoded == 0) { - break; - } - - buffer.insert(buffer.end(), temp.data(), temp.data() + tempFramesDecoded * numChannels); - outFramesRead += tempFramesDecoded; - } - - return buffer; -} - -std::shared_ptr -AudioDecoder::makeAudioBusFromInt16Buffer(const std::vector &buffer, int numChannels, float sampleRate) -{ - auto outputFrames = buffer.size() / numChannels; - auto audioBus = std::make_shared(outputFrames, numChannels, sampleRate); - - for (int ch = 0; ch < numChannels; ++ch) { - auto channelData = audioBus->getChannel(ch)->getData(); - for (int i = 0; i < outputFrames; ++i) { - channelData[i] = int16ToFloat(buffer[i * numChannels + ch]); - } - } - return audioBus; -} - -std::shared_ptr AudioDecoder::decodeWithFilePath(const std::string &path) const -{ - std::vector buffer; - if (AudioDecoder::pathHasExtension(path, {".mp4", ".m4a", ".aac"})) { - buffer = ffmpegdecoding::decodeWithFilePath(path, numChannels_, static_cast(sampleRate_)); - if (buffer.empty()) { - NSLog(@"Failed to decode with FFmpeg: %s", path.c_str()); - return nullptr; - } - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); - } - ma_decoding_backend_vtable *customBackends[] = {ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; - - ma_decoder decoder; - ma_decoder_config config = ma_decoder_config_init(ma_format_s16, numChannels_, static_cast(sampleRate_)); - config.ppCustomBackendVTables = customBackends; - config.customBackendCount = sizeof(customBackends) / sizeof(customBackends[0]); - - if (ma_decoder_init_file(path.c_str(), &config, &decoder) != MA_SUCCESS) { - NSLog(@"Failed to initialize decoder for file: %s", path.c_str()); - ma_decoder_uninit(&decoder); - return nullptr; - } - - ma_uint64 framesRead = 0; - buffer = readAllPcmFrames(decoder, numChannels_, framesRead); - if (framesRead == 0) { - NSLog(@"Failed to decode"); - ma_decoder_uninit(&decoder); - return nullptr; - } - - ma_decoder_uninit(&decoder); - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); -} - -std::shared_ptr AudioDecoder::decodeWithMemoryBlock(const void *data, size_t size) const -{ - std::vector buffer; - const AudioFormat format = AudioDecoder::detectAudioFormat(data, size); - if (format == AudioFormat::MP4 || format == AudioFormat::M4A || format == AudioFormat::AAC) { - buffer = ffmpegdecoding::decodeWithMemoryBlock(data, size, numChannels_, static_cast(sampleRate_)); - if (buffer.empty()) { - NSLog(@"Failed to decode with FFmpeg"); - return nullptr; - } - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); - } - ma_decoding_backend_vtable *customBackends[] = {ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; - - ma_decoder decoder; - ma_decoder_config config = ma_decoder_config_init(ma_format_s16, numChannels_, static_cast(sampleRate_)); - config.ppCustomBackendVTables = customBackends; - config.customBackendCount = sizeof(customBackends) / sizeof(customBackends[0]); - - if (ma_decoder_init_memory(data, size, &config, &decoder) != MA_SUCCESS) { - NSLog(@"Failed to initialize decoder for memory block"); - ma_decoder_uninit(&decoder); - return nullptr; - } - - ma_uint64 framesRead = 0; - buffer = readAllPcmFrames(decoder, numChannels_, framesRead); - if (framesRead == 0) { - NSLog(@"Failed to decode"); - ma_decoder_uninit(&decoder); - return nullptr; - } - - ma_decoder_uninit(&decoder); - return makeAudioBusFromInt16Buffer(buffer, numChannels_, sampleRate_); -} - -std::shared_ptr AudioDecoder::decodeWithPCMInBase64(const std::string &data, float playbackSpeed) const -{ - auto decodedData = base64_decode(data, false); - - const auto uint8Data = reinterpret_cast(decodedData.data()); - size_t framesDecoded = decodedData.size() / 2; - - std::vector buffer(framesDecoded); - for (size_t i = 0; i < framesDecoded; ++i) { - buffer[i] = static_cast((uint8Data[i * 2 + 1] << 8) | uint8Data[i * 2]); - } - - changePlaybackSpeedIfNeeded(buffer, framesDecoded, 1, playbackSpeed); - auto outputFrames = buffer.size(); - - auto audioBus = std::make_shared(outputFrames, numChannels_, sampleRate_); - auto leftChannelData = audioBus->getChannel(0)->getData(); - auto rightChannelData = audioBus->getChannel(1)->getData(); - - for (size_t i = 0; i < outputFrames; ++i) { - auto sample = int16ToFloat(buffer[i]); - leftChannelData[i] = sample; - rightChannelData[i] = sample; - } - - return audioBus; -} - -} // namespace audioapi diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/utils/AudioDecoder.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/utils/AudioDecoder.mm new file mode 100644 index 000000000..9c47fca51 --- /dev/null +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/utils/AudioDecoder.mm @@ -0,0 +1,160 @@ +#define MINIAUDIO_IMPLEMENTATION +#import + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace audioapi { + +// Decoding audio in fixed-size chunks because total frame count can't be +// determined in advance. Note: ma_decoder_get_length_in_pcm_frames() always +// returns 0 for Vorbis decoders. +std::vector AudioDecoder::readAllPcmFrames(ma_decoder &decoder, int outputChannels) +{ + std::vector buffer; + std::vector temp(CHUNK_SIZE * outputChannels); + ma_uint64 outFramesRead = 0; + + while (true) { + ma_uint64 tempFramesDecoded = 0; + ma_decoder_read_pcm_frames(&decoder, temp.data(), CHUNK_SIZE, &tempFramesDecoded); + if (tempFramesDecoded == 0) { + break; + } + + buffer.insert(buffer.end(), temp.data(), temp.data() + tempFramesDecoded * outputChannels); + outFramesRead += tempFramesDecoded; + } + + if (outFramesRead == 0) { + NSLog(@"Failed to decode"); + } + return buffer; +} + +std::shared_ptr AudioDecoder::makeAudioBufferFromFloatBuffer( + const std::vector &buffer, + float outputSampleRate, + int outputChannels) +{ + if (buffer.empty()) { + return nullptr; + } + + auto outputFrames = buffer.size() / outputChannels; + auto audioBus = std::make_shared(outputFrames, outputChannels, outputSampleRate); + + for (int ch = 0; ch < outputChannels; ++ch) { + auto channelData = audioBus->getChannel(ch)->getData(); + for (int i = 0; i < outputFrames; ++i) { + channelData[i] = buffer[i * outputChannels + ch]; + } + } + return std::make_shared(audioBus); +} + +std::shared_ptr AudioDecoder::decodeWithFilePath(const std::string &path, float sampleRate) +{ + if (AudioDecoder::pathHasExtension(path, {".mp4", ".m4a", ".aac"})) { + auto buffer = ffmpegdecoder::decodeWithFilePath(path, static_cast(sampleRate)); + if (buffer == nullptr) { + NSLog(@"Failed to decode with FFmpeg: %s", path.c_str()); + return nullptr; + } + return buffer; + } + ma_decoder decoder; + ma_decoder_config config = ma_decoder_config_init(ma_format_f32, 0, static_cast(sampleRate)); + ma_decoding_backend_vtable *customBackends[] = {ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; + + config.ppCustomBackendVTables = customBackends; + config.customBackendCount = sizeof(customBackends) / sizeof(customBackends[0]); + + if (ma_decoder_init_file(path.c_str(), &config, &decoder) != MA_SUCCESS) { + NSLog(@"Failed to initialize decoder for file: %s", path.c_str()); + ma_decoder_uninit(&decoder); + return nullptr; + } + + auto outputSampleRate = static_cast(decoder.outputSampleRate); + auto outputChannels = static_cast(decoder.outputChannels); + + std::vector buffer = readAllPcmFrames(decoder, outputChannels); + ma_decoder_uninit(&decoder); + return makeAudioBufferFromFloatBuffer(buffer, outputSampleRate, outputChannels); +} + +std::shared_ptr AudioDecoder::decodeWithMemoryBlock(const void *data, size_t size, float sampleRate) +{ + const AudioFormat format = AudioDecoder::detectAudioFormat(data, size); + if (format == AudioFormat::MP4 || format == AudioFormat::M4A || format == AudioFormat::AAC) { + auto buffer = ffmpegdecoder::decodeWithMemoryBlock(data, size, static_cast(sampleRate)); + if (buffer == nullptr) { + NSLog(@"Failed to decode with FFmpeg"); + return nullptr; + } + return buffer; + } + ma_decoder decoder; + ma_decoder_config config = ma_decoder_config_init(ma_format_f32, 0, static_cast(sampleRate)); + + ma_decoding_backend_vtable *customBackends[] = {ma_decoding_backend_libvorbis, ma_decoding_backend_libopus}; + + config.ppCustomBackendVTables = customBackends; + config.customBackendCount = sizeof(customBackends) / sizeof(customBackends[0]); + + if (ma_decoder_init_memory(data, size, &config, &decoder) != MA_SUCCESS) { + NSLog(@"Failed to initialize decoder for memory block"); + ma_decoder_uninit(&decoder); + return nullptr; + } + + auto outputSampleRate = static_cast(decoder.outputSampleRate); + auto outputChannels = static_cast(decoder.outputChannels); + + std::vector buffer = readAllPcmFrames(decoder, outputChannels); + ma_decoder_uninit(&decoder); + return makeAudioBufferFromFloatBuffer(buffer, outputSampleRate, outputChannels); +} + +std::shared_ptr AudioDecoder::decodeWithPCMInBase64( + const std::string &data, + float inputSampleRate, + int inputChannelCount, + bool interleaved) +{ + auto decodedData = base64_decode(data, false); + const auto uint8Data = reinterpret_cast(decodedData.data()); + size_t numFramesDecoded = decodedData.size() / (inputChannelCount * sizeof(int16_t)); + + auto audioBus = std::make_shared(numFramesDecoded, inputChannelCount, inputSampleRate); + + for (int ch = 0; ch < inputChannelCount; ++ch) { + auto channelData = audioBus->getChannel(ch)->getData(); + + for (size_t i = 0; i < numFramesDecoded; ++i) { + size_t offset; + if (interleaved) { + // Ch1, Ch2, Ch1, Ch2, ... + offset = (i * inputChannelCount + ch) * sizeof(int16_t); + } else { + // Ch1, Ch1, Ch1, ..., Ch2, Ch2, Ch2, ... + offset = (ch * numFramesDecoded + i) * sizeof(int16_t); + } + + channelData[i] = uint8ToFloat(uint8Data[offset], uint8Data[offset + 1]); + } + } + return std::make_shared(audioBus); +} + +} // namespace audioapi diff --git a/packages/react-native-audio-api/src/api.ts b/packages/react-native-audio-api/src/api.ts index b62d8fbf3..14b428de0 100644 --- a/packages/react-native-audio-api/src/api.ts +++ b/packages/react-native-audio-api/src/api.ts @@ -2,6 +2,7 @@ import { NativeAudioAPIModule } from './specs'; import { AudioRecorderOptions } from './types'; import type { IAudioContext, + IAudioDecoder, IAudioRecorder, IOfflineAudioContext, IAudioEventEmitter, @@ -25,6 +26,8 @@ declare global { var createAudioRecorder: (options: AudioRecorderOptions) => IAudioRecorder; + var createAudioDecoder: () => IAudioDecoder; + var AudioEventEmitter: IAudioEventEmitter; } /* eslint-disable no-var */ @@ -33,6 +36,7 @@ if ( global.createAudioContext == null || global.createOfflineAudioContext == null || global.createAudioRecorder == null || + global.createAudioDecoder == null || global.AudioEventEmitter == null ) { if (!NativeAudioAPIModule) { @@ -68,6 +72,7 @@ export { default as StreamerNode } from './core/StreamerNode'; export { default as ConstantSourceNode } from './core/ConstantSourceNode'; export { default as AudioManager } from './system'; export { default as useSystemVolume } from './hooks/useSystemVolume'; +export { decodeAudioData, decodePCMInBase64 } from './core/AudioDecoder'; export { OscillatorType, diff --git a/packages/react-native-audio-api/src/core/AudioDecoder.ts b/packages/react-native-audio-api/src/core/AudioDecoder.ts new file mode 100644 index 000000000..367f8f885 --- /dev/null +++ b/packages/react-native-audio-api/src/core/AudioDecoder.ts @@ -0,0 +1,78 @@ +import { IAudioDecoder } from '../interfaces'; +import AudioBuffer from './AudioBuffer'; + +class AudioDecoder { + private static instance: AudioDecoder | null = null; + protected readonly decoder: IAudioDecoder; + + private constructor() { + this.decoder = global.createAudioDecoder(); + } + + public static getInstance(): AudioDecoder { + if (!AudioDecoder.instance) { + AudioDecoder.instance = new AudioDecoder(); + } + return AudioDecoder.instance; + } + + public async decodeAudioDataInstance( + input: string | ArrayBuffer, + sampleRate?: number + ): Promise { + let buffer; + if (typeof input === 'string') { + // Remove the file:// prefix if it exists + if (input.startsWith('file://')) { + input = input.replace('file://', ''); + } + buffer = await this.decoder.decodeWithFilePath(input, sampleRate ?? 0); + } else if (input instanceof ArrayBuffer) { + buffer = await this.decoder.decodeWithMemoryBlock( + new Uint8Array(input), + sampleRate ?? 0 + ); + } + + if (!buffer) { + throw new Error('Unsupported input type or failed to decode audio'); + } + return new AudioBuffer(buffer); + } + + public async decodePCMInBase64Instance( + base64String: string, + inputSampleRate: number, + inputChannelCount: number, + interleaved: boolean + ): Promise { + const buffer = await this.decoder.decodeWithPCMInBase64( + base64String, + inputSampleRate, + inputChannelCount, + interleaved + ); + return new AudioBuffer(buffer); + } +} + +export async function decodeAudioData( + input: string | ArrayBuffer, + sampleRate?: number +): Promise { + return AudioDecoder.getInstance().decodeAudioDataInstance(input, sampleRate); +} + +export async function decodePCMInBase64( + base64String: string, + inputSampleRate: number, + inputChannelCount: number, + isInterleaved: boolean = true +): Promise { + return AudioDecoder.getInstance().decodePCMInBase64Instance( + base64String, + inputSampleRate, + inputChannelCount, + isInterleaved + ); +} diff --git a/packages/react-native-audio-api/src/core/BaseAudioContext.ts b/packages/react-native-audio-api/src/core/BaseAudioContext.ts index 12c40ae1a..8482ac61e 100644 --- a/packages/react-native-audio-api/src/core/BaseAudioContext.ts +++ b/packages/react-native-audio-api/src/core/BaseAudioContext.ts @@ -15,6 +15,7 @@ import AudioBufferQueueSourceNode from './AudioBufferQueueSourceNode'; import AudioBufferSourceNode from './AudioBufferSourceNode'; import AudioDestinationNode from './AudioDestinationNode'; import BiquadFilterNode from './BiquadFilterNode'; +import ConstantSourceNode from './ConstantSourceNode'; import GainNode from './GainNode'; import OscillatorNode from './OscillatorNode'; import PeriodicWave from './PeriodicWave'; @@ -22,7 +23,7 @@ import RecorderAdapterNode from './RecorderAdapterNode'; import StereoPannerNode from './StereoPannerNode'; import StreamerNode from './StreamerNode'; import WorkletNode from './WorkletNode'; -import ConstantSourceNode from './ConstantSourceNode'; +import { decodeAudioData, decodePCMInBase64 } from './AudioDecoder'; export default class BaseAudioContext { readonly destination: AudioDestinationNode; @@ -43,6 +44,30 @@ export default class BaseAudioContext { return this.context.state; } + public async decodeAudioData( + input: string | ArrayBuffer, + sampleRate?: number + ): Promise { + if (!(typeof input === 'string' || input instanceof ArrayBuffer)) { + throw new TypeError('Input must be a string or ArrayBuffer'); + } + return await decodeAudioData(input, sampleRate ?? this.sampleRate); + } + + public async decodePCMInBase64( + base64String: string, + inputSampleRate: number, + inputChannelCount: number, + isInterleaved: boolean = true + ): Promise { + return await decodePCMInBase64( + base64String, + inputSampleRate, + inputChannelCount, + isInterleaved + ); + } + createWorkletNode( callback: (audioData: Array, channelCount: number) => void, bufferLength: number, @@ -264,32 +289,4 @@ export default class BaseAudioContext { createAnalyser(): AnalyserNode { return new AnalyserNode(this, this.context.createAnalyser()); } - - /** Decodes audio data from a local file path. */ - async decodeAudioDataSource(sourcePath: string): Promise { - // Remove the file:// prefix if it exists - if (sourcePath.startsWith('file://')) { - sourcePath = sourcePath.replace('file://', ''); - } - - return new AudioBuffer( - await this.context.decodeAudioDataSource(sourcePath) - ); - } - - /** Decodes audio data from an ArrayBuffer. */ - async decodeAudioData(data: ArrayBuffer): Promise { - return new AudioBuffer( - await this.context.decodeAudioData(new Uint8Array(data)) - ); - } - - async decodePCMInBase64Data( - base64: string, - playbackRate: number = 1.0 - ): Promise { - return new AudioBuffer( - await this.context.decodePCMAudioDataInBase64(base64, playbackRate) - ); - } } diff --git a/packages/react-native-audio-api/src/interfaces.ts b/packages/react-native-audio-api/src/interfaces.ts index 22129d4cc..7b8f70fe2 100644 --- a/packages/react-native-audio-api/src/interfaces.ts +++ b/packages/react-native-audio-api/src/interfaces.ts @@ -37,6 +37,7 @@ export interface IBaseAudioContext { readonly state: ContextState; readonly sampleRate: number; readonly currentTime: number; + readonly decoder: IAudioDecoder; createRecorderAdapter(): IRecorderAdapterNode; createWorkletSourceNode( @@ -73,12 +74,6 @@ export interface IBaseAudioContext { disableNormalization: boolean ) => IPeriodicWave; createAnalyser: () => IAnalyserNode; - decodeAudioDataSource: (sourcePath: string) => Promise; - decodeAudioData: (arrayBuffer: ArrayBuffer) => Promise; - decodePCMAudioDataInBase64: ( - b64: string, - playbackRate: number - ) => Promise; createStreamer: () => IStreamerNode; } @@ -264,6 +259,23 @@ export interface IAudioRecorder { onAudioReady: string; } +export interface IAudioDecoder { + decodeWithMemoryBlock: ( + arrayBuffer: ArrayBuffer, + sampleRate?: number + ) => Promise; + decodeWithFilePath: ( + sourcePath: string, + sampleRate?: number + ) => Promise; + decodeWithPCMInBase64: ( + b64: string, + inputSampleRate: number, + inputChannelCount: number, + interleaved?: boolean + ) => Promise; +} + export interface IAudioEventEmitter { addAudioEventListener( name: Name,