diff --git a/codegen/NativeLocalAISpec.g.h b/codegen/NativeLocalAISpec.g.h new file mode 100644 index 0000000..27581db --- /dev/null +++ b/codegen/NativeLocalAISpec.g.h @@ -0,0 +1,57 @@ +/* + * This file is auto-generated from a NativeModule spec file in js. + * + * This is a C++ Spec class that should be used with MakeTurboModuleProvider to register native modules + * in a way that also verifies at compile time that the native module matches the interface required + * by the TurboModule JS spec. + */ +#pragma once +// clang-format off + +#include +#include + +namespace ArtificialChatModules { + +struct LocalAISpec_LocalAICapabilities { + bool isSupported; + bool hasNPU; + bool hasGPU; + std::optional modelName; +}; + + +inline winrt::Microsoft::ReactNative::FieldMap GetStructInfo(LocalAISpec_LocalAICapabilities*) noexcept { + winrt::Microsoft::ReactNative::FieldMap fieldMap { + {L"isSupported", &LocalAISpec_LocalAICapabilities::isSupported}, + {L"hasNPU", &LocalAISpec_LocalAICapabilities::hasNPU}, + {L"hasGPU", &LocalAISpec_LocalAICapabilities::hasGPU}, + {L"modelName", &LocalAISpec_LocalAICapabilities::modelName}, + }; + return fieldMap; +} + +struct LocalAISpec : winrt::Microsoft::ReactNative::TurboModuleSpec { + static constexpr auto methods = std::tuple{ + SyncMethod{0, L"checkCapabilities"}, + Method, Promise) noexcept>{1, L"generateText"}, + }; + + template + static constexpr void ValidateModule() noexcept { + constexpr auto methodCheckResults = CheckMethods(); + + REACT_SHOW_METHOD_SPEC_ERRORS( + 0, + "checkCapabilities", + " REACT_SYNC_METHOD(checkCapabilities) LocalAISpec_LocalAICapabilities checkCapabilities() noexcept { /* implementation */ }\n" + " REACT_SYNC_METHOD(checkCapabilities) static LocalAISpec_LocalAICapabilities checkCapabilities() noexcept { /* implementation */ }\n"); + REACT_SHOW_METHOD_SPEC_ERRORS( + 1, + "generateText", + " REACT_METHOD(generateText) void generateText(std::string prompt, std::optional systemInstructions, ::React::ReactPromise &&result) noexcept { /* implementation */ }\n" + " REACT_METHOD(generateText) static void generateText(std::string prompt, std::optional systemInstructions, ::React::ReactPromise &&result) noexcept { /* implementation */ }\n"); + } +}; + +} // namespace ArtificialChatModules \ No newline at end of file diff --git a/src/AiQuery.tsx b/src/AiQuery.tsx index b0a64cd..39046d6 100644 --- a/src/AiQuery.tsx +++ b/src/AiQuery.tsx @@ -4,6 +4,10 @@ import { OpenAiApi, CallOpenAi, } from './OpenAI'; +import { + CallLocalAI, + IsLocalAIAvailable, +} from './LocalAI'; import { AiSection } from './AiResponse'; import { ChatSource, @@ -130,36 +134,66 @@ Respond with the image prompt string in the required format. Do not respond conv React.useEffect(() => { if (isRequestForImage === false) { setIsLoading(true); - CallOpenAi({ - api: OpenAiApi.ChatCompletion, - apiKey: settingsContext.apiKey, - instructions: settingsContext.systemInstructions, - identifier: 'TEXT-ANSWER:', - prompt: prompt, - options: { - endpoint: settingsContext.aiEndpoint, - chatModel: settingsContext.chatModel, - promptHistory: chatHistory.entries. - filter((entry) => { return entry.responses !== undefined && entry.id < id; }). - map((entry) => { return {role: entry.type == ChatSource.Human ? 'user' : 'assistant', 'content': entry.responses ? entry.responses[0] : ''}; }), - }, - onError: error => { - onResponse({ - prompt: prompt, - responses: [error] ?? [''], - contentType: ChatContent.Error}); - }, - onResult: result => { - onResponse({ - prompt: prompt, - responses: result ?? [''], - contentType: ChatContent.Text}); - }, - onComplete: () => { - setIsLoading(false); - chatScroll.scrollToEnd(); - }, - }); + + // Check if user prefers local AI and it's available + const shouldUseLocalAI = settingsContext.useLocalAI && IsLocalAIAvailable(); + + if (shouldUseLocalAI) { + // Use local AI for text generation + CallLocalAI({ + instructions: settingsContext.systemInstructions, + identifier: 'LOCAL-TEXT-ANSWER:', + prompt: prompt, + onError: error => { + onResponse({ + prompt: prompt, + responses: [error] ?? [''], + contentType: ChatContent.Error}); + }, + onResult: result => { + onResponse({ + prompt: prompt, + responses: result ?? [''], + contentType: ChatContent.Text}); + }, + onComplete: () => { + setIsLoading(false); + chatScroll.scrollToEnd(); + }, + }); + } else { + // Use OpenAI for text generation + CallOpenAi({ + api: OpenAiApi.ChatCompletion, + apiKey: settingsContext.apiKey, + instructions: settingsContext.systemInstructions, + identifier: 'TEXT-ANSWER:', + prompt: prompt, + options: { + endpoint: settingsContext.aiEndpoint, + chatModel: settingsContext.chatModel, + promptHistory: chatHistory.entries. + filter((entry) => { return entry.responses !== undefined && entry.id < id; }). + map((entry) => { return {role: entry.type == ChatSource.Human ? 'user' : 'assistant', 'content': entry.responses ? entry.responses[0] : ''}; }), + }, + onError: error => { + onResponse({ + prompt: prompt, + responses: [error] ?? [''], + contentType: ChatContent.Error}); + }, + onResult: result => { + onResponse({ + prompt: prompt, + responses: result ?? [''], + contentType: ChatContent.Text}); + }, + onComplete: () => { + setIsLoading(false); + chatScroll.scrollToEnd(); + }, + }); + } } else { if (isRequestForImage == true && imagePrompt !== undefined) { setIsLoading(true); @@ -206,7 +240,11 @@ Respond with the image prompt string in the required format. Do not respond conv Generating image... ) ) : ( - Generating text... + + {settingsContext.useLocalAI && IsLocalAIAvailable() + ? 'Generating text using local AI...' + : 'Generating text...'} + ) ) : ( Done loading diff --git a/src/App.tsx b/src/App.tsx index 3313ff7..83a8e8e 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -27,6 +27,7 @@ function App(): JSX.Element { const [showAboutPopup, setShowAboutPopup] = React.useState(false); const [readToMeVoice, setReadToMeVoice] = React.useState(''); const [systemInstructions, setSystemInstructions] = React.useState('The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly. You may use markdown syntax in the response as appropriate.'); + const [useLocalAI, setUseLocalAI] = React.useState(false); const isDarkMode = currentTheme === 'dark'; const isHighContrast = false; @@ -49,6 +50,8 @@ function App(): JSX.Element { setReadToMeVoice: setReadToMeVoice, systemInstructions: systemInstructions, setSystemInstructions: setSystemInstructions, + useLocalAI: useLocalAI, + setUseLocalAI: setUseLocalAI, }; const popups = { diff --git a/src/LocalAI.tsx b/src/LocalAI.tsx new file mode 100644 index 0000000..78206ab --- /dev/null +++ b/src/LocalAI.tsx @@ -0,0 +1,81 @@ +import NativeLocalAI from './NativeLocalAI'; + +type CallLocalAIType = { + instructions?: string; + identifier?: string; + prompt: string; + onError: (error: string) => void; + onResult: (results: string[]) => void; + onComplete: () => void; +}; + +const CallLocalAI = async ({ + instructions, + identifier, + prompt, + onError, + onResult, + onComplete, +}: CallLocalAIType) => { + try { + if (!NativeLocalAI) { + onError('Local AI is not available on this platform'); + onComplete(); + return; + } + + // Check if local AI is supported + const capabilities = NativeLocalAI.checkCapabilities(); + if (!capabilities.isSupported) { + onError('Local AI is not supported on this device. Compatible NPU/GPU hardware required.'); + onComplete(); + return; + } + + console.debug(`Start LocalAI ${identifier}"${prompt}"`); + + const actualInstructions = + instructions ?? + 'The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly. You may use markdown syntax in the response as appropriate.'; + + const result = await NativeLocalAI.generateText(prompt, actualInstructions); + + console.log(`LocalAI response: "${result}"`); + onResult([result]); + } catch (error) { + console.error('LocalAI error:', error); + onError(error instanceof Error ? error.message : 'Error generating local AI response'); + } finally { + console.debug(`End LocalAI ${identifier}"${prompt}"`); + onComplete(); + } +}; + +// Function to check if local AI is available +const IsLocalAIAvailable = (): boolean => { + if (!NativeLocalAI) { + return false; + } + + try { + const capabilities = NativeLocalAI.checkCapabilities(); + return capabilities.isSupported; + } catch { + return false; + } +}; + +// Function to get local AI capabilities info +const GetLocalAICapabilities = () => { + if (!NativeLocalAI) { + return { isSupported: false, hasNPU: false, hasGPU: false }; + } + + try { + return NativeLocalAI.checkCapabilities(); + } catch { + return { isSupported: false, hasNPU: false, hasGPU: false }; + } +}; + +export { CallLocalAI, IsLocalAIAvailable, GetLocalAICapabilities }; \ No newline at end of file diff --git a/src/NativeLocalAI.ts b/src/NativeLocalAI.ts new file mode 100644 index 0000000..dddd510 --- /dev/null +++ b/src/NativeLocalAI.ts @@ -0,0 +1,18 @@ +import type { TurboModule } from 'react-native'; +import { TurboModuleRegistry } from 'react-native'; + +export interface LocalAICapabilities { + isSupported: boolean; + hasNPU: boolean; + hasGPU: boolean; + modelName?: string; +} + +export interface Spec extends TurboModule { + checkCapabilities(): LocalAICapabilities; + generateText(prompt: string, systemInstructions?: string): Promise; +} + +export default TurboModuleRegistry.get( + 'LocalAI' +) as Spec | null; \ No newline at end of file diff --git a/src/Settings.tsx b/src/Settings.tsx index 6e7573c..aaf8991 100644 --- a/src/Settings.tsx +++ b/src/Settings.tsx @@ -11,6 +11,7 @@ import { } from './FluentControls'; import { GetVoices, SetVoice } from './Speech'; import { getRemainingTrialUses, MAX_TRIAL_USES } from './TrialMode'; +import { GetLocalAICapabilities } from './LocalAI'; const settingsKey = 'settings'; @@ -32,6 +33,8 @@ type SettingsContextType = { setReadToMeVoice: (value: string) => void; systemInstructions: string; setSystemInstructions: (value: string) => void; + useLocalAI: boolean; + setUseLocalAI: (value: boolean) => void; }; const SettingsContext = React.createContext({ setApiKey: () => {}, @@ -49,6 +52,8 @@ const SettingsContext = React.createContext({ setReadToMeVoice: () => {}, systemInstructions: '', setSystemInstructions: () => {}, + useLocalAI: false, + setUseLocalAI: () => {}, }); // Settings that are saved between app sessions @@ -57,6 +62,7 @@ type SettingsData = { imageSize?: number; readToMeVoice?: string; systemInstructions?: string; + useLocalAI?: boolean; }; // Read settings from app storage @@ -84,6 +90,7 @@ const LoadSettingsData = async () => { if (value.hasOwnProperty('imageSize')) { valueToSave.imageSize = parseInt(value.imageSize, 10); } if (value.hasOwnProperty('readToMeVoice')) { valueToSave.readToMeVoice = value.readToMeVoice; } if (value.hasOwnProperty('systemInstructions')) { valueToSave.systemInstructions = value.systemInstructions; } + if (value.hasOwnProperty('useLocalAI')) { valueToSave.useLocalAI = value.useLocalAI; } } } catch (e) { console.error(e); @@ -115,6 +122,9 @@ function SettingsPopup({show, close}: SettingsPopupProps): JSX.Element { const [systemInstructions, setSystemInstructions] = React.useState( settings.systemInstructions, ); + const [useLocalAI, setUseLocalAI] = React.useState( + settings.useLocalAI, + ); // Trial mode state const [remainingTrialUses, setRemainingTrialUses] = React.useState(0); @@ -174,6 +184,7 @@ function SettingsPopup({show, close}: SettingsPopupProps): JSX.Element { settings.setImageSize(imageSize); settings.setReadToMeVoice(readToMeVoice); settings.setSystemInstructions(systemInstructions); + settings.setUseLocalAI(useLocalAI); close(); @@ -185,6 +196,7 @@ function SettingsPopup({show, close}: SettingsPopupProps): JSX.Element { imageSize: imageSize, readToMeVoice: readToMeVoice, systemInstructions: systemInstructions, + useLocalAI: useLocalAI, }); }; @@ -197,6 +209,7 @@ function SettingsPopup({show, close}: SettingsPopupProps): JSX.Element { setImageSize(settings.imageSize); setReadToMeVoice(settings.readToMeVoice); setSystemInstructions(settings.systemInstructions); + setUseLocalAI(settings.useLocalAI); close(); }; @@ -283,6 +296,24 @@ function SettingsPopup({show, close}: SettingsPopupProps): JSX.Element { url="https://platform.openai.com/account/api-keys" /> + + setUseLocalAI(value)} + /> + + {(() => { + const capabilities = GetLocalAICapabilities(); + if (!capabilities.isSupported) { + return 'Local AI is not supported on this device. Compatible NPU/GPU hardware required.'; + } else { + return `Local AI is available${capabilities.modelName ? ` (${capabilities.modelName})` : ''}. Enable to use local hardware for text generation instead of cloud APIs.`; + } + })()} + + +#include +#include + +/* + * LocalAI TurboModule Implementation + * + * This module provides local NPU/GPU-accelerated text generation using Microsoft's Phi Silica model + * through the Windows AI Foundry APIs. It enables the app to perform AI text generation locally + * instead of relying on cloud-based API calls when compatible hardware is available. + * + * Hardware Requirements: + * - Windows 11 (22H2 or later) + * - Compatible NPU (Neural Processing Unit) - typically found in CoPilot+ PCs + * - Windows App SDK 1.8 or later + * + * Features: + * - Automatic hardware capability detection + * - Local Phi Silica model inference + * - Graceful fallback when hardware is unsupported + * - System instruction support for consistent AI behavior + * - Proper error handling and user feedback + */ + +namespace ArtificialChatModules +{ + REACT_MODULE(LocalAI); + struct LocalAI + { + using ModuleSpec = LocalAISpec; + + LocalAI() + { + // Initialize will be done lazily when needed + } + + // Synchronously check if local AI capabilities are available on this device + REACT_SYNC_METHOD(checkCapabilities) + LocalAISpec_LocalAICapabilities checkCapabilities() noexcept + { + LocalAISpec_LocalAICapabilities capabilities; + + try + { + // Check if Phi Silica language model is available + // This will only succeed on compatible CoPilot+ PCs with proper NPU support + auto languageModel = winrt::Microsoft::Windows::AI::Text::LanguageModel::GetDefault(); + + if (languageModel) + { + capabilities.isSupported = true; + capabilities.hasNPU = true; // Assume NPU if Phi Silica is available + capabilities.hasGPU = false; // Phi Silica primarily uses NPU + capabilities.modelName = "Phi Silica"; + } + else + { + capabilities.isSupported = false; + capabilities.hasNPU = false; + capabilities.hasGPU = false; + capabilities.modelName = std::nullopt; + } + } + catch (...) + { + // If any error occurs (e.g., API not available, hardware not supported), assume not supported + capabilities.isSupported = false; + capabilities.hasNPU = false; + capabilities.hasGPU = false; + capabilities.modelName = std::nullopt; + } + + return capabilities; + } + + // Asynchronously generate text using local Phi Silica model + REACT_METHOD(generateText) + winrt::fire_and_forget generateText(std::string prompt, std::optional systemInstructions, ::React::ReactPromise result) noexcept + { + try + { + // Get the default Phi Silica language model + auto languageModel = winrt::Microsoft::Windows::AI::Text::LanguageModel::GetDefault(); + + if (!languageModel) + { + result.Reject("Phi Silica language model is not available on this device. Compatible NPU hardware and Windows AI support required."); + co_return; + } + + // Prepare the full prompt with system instructions + // Format: [System Instructions]\n\nHuman: [User Prompt]\n\nAssistant: + std::string fullPrompt; + if (systemInstructions.has_value() && !systemInstructions.value().empty()) + { + fullPrompt = systemInstructions.value() + "\n\nHuman: " + prompt + "\n\nAssistant:"; + } + else + { + fullPrompt = "Human: " + prompt + "\n\nAssistant:"; + } + + // Convert to winrt::hstring for Windows API + winrt::hstring wprompt = winrt::to_hstring(fullPrompt); + + // Generate response using Phi Silica - this runs on the NPU + auto response = co_await languageModel.GenerateResponseAsync(wprompt); + + // Convert response back to std::string + std::string responseText = winrt::to_string(response); + + // Clean up the response (remove any prompt echo that might be included) + size_t assistantPos = responseText.find("Assistant:"); + if (assistantPos != std::string::npos) + { + responseText = responseText.substr(assistantPos + 10); // Skip "Assistant:" + } + + // Trim whitespace from both ends + auto start = responseText.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) + { + responseText = ""; + } + else + { + auto end = responseText.find_last_not_of(" \t\n\r"); + responseText = responseText.substr(start, end - start + 1); + } + + // Ensure we have a non-empty response + if (responseText.empty()) + { + responseText = "I apologize, but I couldn't generate a response. Please try again."; + } + + result.Resolve(responseText); + } + catch (winrt::hresult_error const& ex) + { + // Handle Windows-specific errors with detailed error information + std::wstring errorMsg = L"Error generating text with Phi Silica: "; + errorMsg += ex.message(); + result.Reject(winrt::to_string(errorMsg)); + } + catch (...) + { + // Handle any other unexpected errors + result.Reject("Unexpected error occurred while generating text with local AI model"); + } + } + + private: + // No persistent state needed - Phi Silica model is managed by the Windows AI system + }; +} \ No newline at end of file diff --git a/windows/artificialChat/artificialChat.cpp b/windows/artificialChat/artificialChat.cpp index 0a996a5..9ffd1d1 100644 --- a/windows/artificialChat/artificialChat.cpp +++ b/windows/artificialChat/artificialChat.cpp @@ -7,6 +7,7 @@ #include "AutolinkedNativeModules.g.h" #include "VersionInfo.h" #include "Speech.h" +#include "LocalAI.h" #include "NativeModules.h" @@ -18,6 +19,7 @@ struct CompReactPackageProvider AddAttributedModules(packageBuilder, true); packageBuilder.AddModule(L"VersionInfo", winrt::Microsoft::ReactNative::MakeTurboModuleProvider()); packageBuilder.AddModule(L"Speech", winrt::Microsoft::ReactNative::MakeTurboModuleProvider()); + packageBuilder.AddModule(L"LocalAI", winrt::Microsoft::ReactNative::MakeTurboModuleProvider()); } }; diff --git a/windows/artificialChat/artificialChat.vcxproj b/windows/artificialChat/artificialChat.vcxproj index 1c1fe39..baea22a 100644 --- a/windows/artificialChat/artificialChat.vcxproj +++ b/windows/artificialChat/artificialChat.vcxproj @@ -65,6 +65,8 @@ + + @@ -105,6 +107,7 @@ + @@ -132,6 +135,8 @@ + + @@ -147,5 +152,7 @@ This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + diff --git a/windows/artificialChat/packages.config b/windows/artificialChat/packages.config index afb3f57..1c8bf32 100644 --- a/windows/artificialChat/packages.config +++ b/windows/artificialChat/packages.config @@ -2,4 +2,6 @@ + + \ No newline at end of file