Skip to content

Commit 333e0a3

Browse files
r2d4r2d4
committed
initial commit
0 parents  commit 333e0a3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+20831
-0
lines changed

‎.github/workflows/release.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Publish Package to npmjs
2+
on:
3+
release:
4+
types: [published]
5+
jobs:
6+
build:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: actions/checkout@v3
10+
- uses: actions/setup-node@v3
11+
with:
12+
node-version: "16.x"
13+
registry-url: "https://registry.npmjs.org"
14+
- uses: pnpm/action-setup@v2
15+
name: Install pnpm
16+
id: pnpm-install
17+
with:
18+
version: 8
19+
run_install: false
20+
21+
- name: Get pnpm store directory
22+
id: pnpm-cache
23+
shell: bash
24+
run: |
25+
echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
26+
27+
- uses: actions/cache@v3
28+
name: Setup pnpm cache
29+
with:
30+
path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
31+
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
32+
restore-keys: |
33+
${{ runner.os }}-pnpm-store-
34+
35+
- name: Install dependencies
36+
run: pnpm install
37+
38+
- name: Build
39+
run: pnpm build
40+
- name: Publish to npmjs
41+
run: pnpm publish
42+
env:
43+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
node_modules
2+
**/*.tsbuildinfo
3+
**/**/.next
4+
**/**/node_modules

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2023 Matt Rickard
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎README.md‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# @react-llm/headless
2+
3+
Easy-to-use headless React Hooks to run LLMs in the browser with WebGPU. As simple as `useLLM()`.
4+
5+
### [**Live Demo**](https://chat.matt-rickard.com)
6+
7+
![image](assets/demo.webp)
8+
9+
**Features**:
10+
11+
* Supports [Vicuna 13B](https://lmsys.org/blog/2023-03-30-vicuna/)
12+
* Use custom system prompts and "user:"/"assistant:" role names
13+
* Completion options like `max tokens` and `stop sequences`
14+
* No data leaves the browser. Accelerated via WebGPU.
15+
* Hooks built to 'Bring your own UI'
16+
* Persistent storage for conversations in browser storage. Hooks for loading and saving conversations.
17+
* Model caching for faster subsequent loads
18+
19+
## Installation
20+
21+
```bash
22+
npm install @react-llm/headless
23+
```
24+
25+
26+
## **useLLM** API
27+
### Types
28+
```typescript
29+
// Model Initialization
30+
init: () => void;
31+
32+
// Model Generation
33+
send: (msg: string, maxTokens: number, stopSequences: string[]) => void;
34+
onMessage: (msg: GenerateTextResponse) => void;
35+
setOnMessage: (cb: (msg: GenerateTextResponse) => void) => void;
36+
37+
// Model Status
38+
loadingStatus: InitProgressReport;
39+
isGenerating: boolean;
40+
gpuDevice: GPUDeviceInfo;
41+
42+
// Model Configuration
43+
userRoleName: string;
44+
setUserRoleName: (roleName: string) => void;
45+
assistantRoleName: string;
46+
setAssistantRoleName: (roleName: string) => void;
47+
48+
// Conversation Management
49+
conversation: Conversation | undefined;
50+
allConversations: Conversation[] | undefined;
51+
createConversation: (title?: string, prompt?: string) => void;
52+
setConversationId: (conversationId: string) => void;
53+
deleteConversation: (conversationId: string) => void;
54+
deleteAllConversations: () => void;
55+
deleteMessages: () => void;
56+
setConversationTitle: (conversationId: string, title: string) => void;
57+
```
58+
59+
### Hooks
60+
```typescript
61+
import useLLM from '@react-llm/headless';
62+
63+
const MyComponent = () => {
64+
const {
65+
conversation,
66+
allConversations,
67+
loadingStatus,
68+
isGenerating,
69+
createConversation,
70+
setConversationId,
71+
deleteConversation,
72+
deleteAllConversations,
73+
deleteMessages,
74+
setConversationTitle,
75+
onMessage,
76+
setOnMessage,
77+
userRoleName,
78+
setUserRoleName,
79+
assistantRoleName,
80+
setAssistantRoleName,
81+
gpuDevice,
82+
send,
83+
init,
84+
} = useLLM();
85+
86+
// Component logic...
87+
88+
return null;
89+
};
90+
```
91+
92+
93+
### Packages
94+
95+
* `@react-llm/headless` - Headless React Hooks for running LLMs in the browser
96+
* `@react-llm/retro-ui` - Retro-themed UI for the hooks
97+
98+
## How does it work?
99+
100+
This library is a set of React Hooks that provide a simple interface to run LLMs in the browser. It uses Vicuna 13B.
101+
102+
* SentencePiece tokenizer (compiled for the browser via Emscripten)
103+
* Vicuna 13B (transformed to Apache TVM format)
104+
* Apache TVM and MLC Relax (compiled for the browser via Emscripten)
105+
* Off-the-main-thread WebWorker to run the model (bundled with the library)
106+
107+
108+
The model, tokenizer, and TVM runtime are loaded from a CDN (huggingface). The model is cached in browser storage for faster subsequent loads.
109+
110+
111+
112+
113+
### Example
114+
See [packages/retro-ui](packages/retro-ui) for the full demo code. This is a simple example of how to use the hooks. To run it, after cloning the repo,
115+
116+
```bash
117+
cd packages/retro-ui
118+
pnpm install
119+
pnpm dev
120+
```
121+
122+
123+
### License
124+
MIT
125+
126+
The code under `packages/headless/worker/lib/tvm` is licensed under Apache 2.0.

‎assets/demo.webp‎

237 KB
Loading

‎package.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"name": "@react-llm/workspace",
3+
"version": "0.0.1",
4+
"type": "module",
5+
"main": "dist/bundle.cjs.js",
6+
"module": "dist/bundle.esm.js",
7+
"author": "Matt Rickard <[email protected]>",
8+
"license": "MIT",
9+
"private": true,
10+
"workspaces": [
11+
"packages/headless",
12+
"packages/retro-ui"
13+
],
14+
"scripts": {
15+
"publish": "pnpm publish --access public",
16+
"build": "pnpm recursive run build"
17+
},
18+
"devDependencies": {
19+
"typescript": "^5.0.4"
20+
},
21+
"dependencies": {
22+
"react95": "^4.0.0",
23+
"styled-components": "^5.3.10"
24+
}
25+
}

‎packages/headless/.eslintrc.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"extends": [
3+
"@typescript-eslint/no-unused-vars"
4+
]
5+
}

‎packages/headless/.gitignore‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2+
3+
# dependencies
4+
/node_modules
5+
/.pnp
6+
.pnp.js
7+
8+
# testing
9+
/coverage
10+
11+
# next.js
12+
/.next/
13+
/out/
14+
15+
# production
16+
/build
17+
18+
# misc
19+
.DS_Store
20+
*.pem
21+
22+
# debug
23+
npm-debug.log*
24+
yarn-debug.log*
25+
yarn-error.log*
26+
27+
# local env files
28+
.env*.local
29+
30+
# vercel
31+
.vercel
32+
33+
# typescript
34+
*.tsbuildinfo
35+
next-env.d.ts

‎packages/headless/LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2023 Matt Rickard
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

0 commit comments

Comments
 (0)