From a4385c092afda23c590189e18dfd557d3c86e363 Mon Sep 17 00:00:00 2001
From: Sunset Mikoto <26019675+lwd-temp@users.noreply.github.com>
Date: Thu, 16 May 2024 13:42:00 +0800
Subject: [PATCH] feat: support d1 and combine with kv
---
README.md | 16 ++--
frontend/index.html | 2 +-
schema.sql | 13 +++
src/db.js | 166 +++++++++++++++++++++++++++++++++++
src/handlers/handleDelete.js | 7 +-
src/handlers/handleRead.js | 5 +-
src/handlers/handleWrite.js | 19 ++--
src/pages/highlight.js | 10 +--
src/pages/markdown.js | 12 +--
wrangler.toml | 10 ++-
10 files changed, 227 insertions(+), 33 deletions(-)
create mode 100644 schema.sql
create mode 100644 src/db.js
diff --git a/README.md b/README.md
index 936b6631..14d22bda 100644
--- a/README.md
+++ b/README.md
@@ -25,8 +25,10 @@ This is a pastebin that can be deployed on Cloudflare workers. Try it on [shz.al
## Limitations
-1. If deployed on Cloudflare Worker free-tier plan, the service allows at most 100,000 reads and 1000 writes, 1000 deletes per day.
-2. Due to the size limit of Cloudflare KV storage, the size of each paste is bounded under 25 MB.
+1. If deployed on Cloudflare Worker free-tier plan, the service allows at most 5 million reads and 100000 writes per day.
+2. Due to the size limit of Cloudflare D1 storage, the size of each basic paste is bounded under 1 MB.
+3. Big files (more than 0.95 MB) are storaged in KV (metadata in D1), the size limit is 25 MB. 100000 reads per day. 1000 writes per day.
+4. To save KV writes, we do not actively delete expired big files in KV, we just overwrite them in need.
## Deploy
@@ -34,13 +36,15 @@ You are free to deploy the pastebin on your own domain if you host your domain o
1. Install `node` and `yarn`.
-2. Create a KV namespace on Cloudflare workers dashboard, remember its ID.
+2. Create a D1 Database on Cloudflare workers dashboard, run `schema.sql` in it, remember its ID. Please notice that executing `schema.sql` in the DB will reset all data in the DB.
-3. Clone the repository and enter the directory. Login to your Cloudflare account with `wrangler login`.
+3. Create a KV on Cloudflare workers dashboard, remember its ID.
-4. Modify entries in `wrangler.toml`. Its comments will tell you how.
+4. Clone the repository and enter the directory. Login to your Cloudflare account with `wrangler login`.
-5. Deploy and enjoy!
+5. Modify entries in `wrangler.toml`. Its comments will tell you how.
+
+6. Deploy and enjoy!
```shell
$ yarn install
diff --git a/frontend/index.html b/frontend/index.html
index 15ab1e53..e4be1193 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -43,7 +43,7 @@
Settings
-
+
diff --git a/schema.sql b/schema.sql
new file mode 100644
index 00000000..abde3f35
--- /dev/null
+++ b/schema.sql
@@ -0,0 +1,13 @@
+-- PB schema
+-- In the old KV design, they use metadata to store password, date, etc.
+-- But there's no such metadata in SQLite (D1).
+-- So we create a new table for them.
+-- short: the short url (string)
+-- content: the paste or file content (blob)
+-- metadata: JSON string of metadata (string)
+DROP TABLE IF EXISTS pastes;
+CREATE TABLE IF NOT EXISTS pastes (
+ short TEXT PRIMARY KEY,
+ content BLOB,
+ metadata TEXT
+)
diff --git a/src/db.js b/src/db.js
new file mode 100644
index 00000000..13812d9b
--- /dev/null
+++ b/src/db.js
@@ -0,0 +1,166 @@
+// Cloudflare D1 SQLite DB and KV for big file storage
+import { WorkerError } from "./common.js"
+
+/**
+ await env.PB.put(short, content, {
+ expirationTtl: expire,
+ postedAt: createDate,
+ passwd: passwd,
+ filename: filename,
+ lastModified: now,
+ })
+*/
+
+export function safeAccess(obj, prop, defaultValue) {
+ try {
+ if (obj && typeof obj === "object" && prop in obj) {
+ return obj[prop]
+ } else {
+ return defaultValue
+ }
+ } catch (e) {
+ return defaultValue
+ }
+}
+
+const expiredItemTemplate = {
+ value: "Expired paste",
+ metadata: {
+ expirationTtl: 0,
+ postedAt: Date.toString(Date.now() / 1000 + 10), // let Client Cache reset in 10 seconds
+ passwd: "",
+ filename: "",
+ lastModified: Date.toString(Date.now() / 1000 + 10),
+ },
+}
+
+const NotExistItemTemplate = {
+ value: "Not found",
+ metadata: {
+ expirationTtl: 0,
+ postedAt: Date.toString(Date.now() / 1000),
+ passwd: "",
+ filename: "",
+ lastModified: Date.toString(Date.now() / 1000),
+ },
+}
+
+function isExpired(item, env) {
+ // return true if item is expired
+ const realMetadata = JSON.parse(item.metadata)
+ // Use postedAt, or use lastModified is existing
+ if (realMetadata.lastModified === undefined) {
+ var lastModified = realMetadata.postedAt
+ } else {
+ var lastModified = realMetadata.lastModified
+ }
+ const lastModifiedUnix = Date.parse(lastModified) / 1000 // In seconds
+ const shouldExpireTime = lastModifiedUnix + realMetadata.expirationTtl
+
+ // if realMetadata.expirationTtl is 0, it should never expire
+ if (realMetadata.expirationTtl === 0) return false
+
+ const nowUnix = Date.now() / 1000
+ return nowUnix > shouldExpireTime
+}
+
+async function KV_Put(short, content, env) {
+ await env.KV.put(short, content)
+}
+
+async function KV_Get(short, env) {
+ const content = await env.KV.get(short, { type: "arrayBuffer" })
+ return new Uint8Array(content)
+}
+
+export async function DB_Put(short, content, metadata, env) {
+ // If content is bigger than 0.95MB, save to KV
+ if (content.length > 1024 * 1024 * 0.95) {
+ await KV_Put(short, content, env)
+ content = "{KV_Storaged_Flag_Attention_DO_NOT_DELETE_OR_MODIFY_THIS_LINE}"
+ }
+
+ return await env.DB.prepare(
+ "INSERT OR REPLACE INTO pastes (short, content, metadata) VALUES (?, ?, ?)",
+ )
+ .bind(short, content, JSON.stringify(metadata))
+ .run() // run and don't return actual result
+}
+
+export async function DB_Get(short, env) {
+ const item_db = await env.DB.prepare("SELECT * FROM pastes WHERE short = ?")
+ .bind(short)
+ .first() // run and return first result
+
+ // Check existence
+ if (!item_db) {
+ return null
+ }
+
+ if (isExpired(item_db, env)) {
+ await DB_Delete(short, env)
+ return null
+ }
+
+ return new Uint8Array(item_db.content)
+}
+
+/**
+ const item = await env.PB.getWithMetadata(short)
+ if (item.value === null) {
+ throw new WorkerError(404, `paste of name '${short}' is not found`)
+ } else {
+ const date = item.metadata?.postedAt
+ if (passwd !== item.metadata?.passwd) {
+ throw new WorkerError(403, `incorrect password for paste '${short}`)
+ } else {
+ return makeResponse(
+ await createPaste(env, content, isPrivate, expirationSeconds, short, date, newPasswd || passwd, filename),
+ )
+ }
+ }
+*/
+
+export async function DB_GetWithMetadata(short, env) {
+ const item_db = await env.DB.prepare("SELECT * FROM pastes WHERE short = ?")
+ .bind(short)
+ .first()
+
+ // Check existence
+ if (!item_db) {
+ // return NotExistItemTemplate
+ return null
+ }
+
+ if (isExpired(item_db, env)) {
+ await DB_Delete(short, env)
+ return expiredItemTemplate // This function is not used to check if the paste exists
+ // So it's okay to return non-null expiredItemTemplate
+ }
+
+ if (
+ item_db.content ===
+ "{KV_Storaged_Flag_Attention_DO_NOT_DELETE_OR_MODIFY_THIS_LINE}"
+ ) {
+ const kv_content = await KV_Get(short, env)
+ var item = {
+ value: kv_content,
+ metadata: JSON.parse(item_db.metadata),
+ }
+ } else {
+ var item = {
+ value: new Uint8Array(item_db.content),
+ metadata: JSON.parse(item_db.metadata),
+ }
+ }
+
+ // console.log(item)
+
+ return item
+}
+
+export async function DB_Delete(short, env) {
+ return await env.DB.prepare("DELETE FROM pastes WHERE short = ?")
+ .bind(short)
+ .run() // Run and don't return actual result
+}
diff --git a/src/handlers/handleDelete.js b/src/handlers/handleDelete.js
index 63dc16af..87f4042d 100644
--- a/src/handlers/handleDelete.js
+++ b/src/handlers/handleDelete.js
@@ -1,16 +1,17 @@
import { parsePath, WorkerError } from "../common.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, DB_Delete, safeAccess } from "../db.js"
export async function handleDelete(request, env, ctx) {
const url = new URL(request.url)
const { short, passwd } = parsePath(url.pathname)
- const item = await env.PB.getWithMetadata(short)
- if (item.value === null) {
+ const item = await DB_GetWithMetadata(short, env)
+ if (safeAccess(item, "value", null) === null) {
throw new WorkerError(404, `paste of name '${short}' not found`)
} else {
if (passwd !== item.metadata?.passwd) {
throw new WorkerError(403, `incorrect password for paste '${short}`)
} else {
- await env.PB.delete(short)
+ await DB_Delete(short, env)
return new Response("the paste will be deleted in seconds")
}
}
diff --git a/src/handlers/handleRead.js b/src/handlers/handleRead.js
index e6a1b94c..fe3e78c3 100644
--- a/src/handlers/handleRead.js
+++ b/src/handlers/handleRead.js
@@ -4,6 +4,7 @@ import { verifyAuth } from "../auth.js"
import { getType } from "mime/lite.js"
import { makeMarkdown } from "../pages/markdown.js"
import { makeHighlight } from "../pages/highlight.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, safeAccess } from "../db.js"
function staticPageCacheHeader(env) {
const age = env.CACHE_STATIC_PAGE_AGE
@@ -45,10 +46,10 @@ export async function handleGet(request, env, ctx) {
const disp = url.searchParams.has("a") ? "attachment" : "inline"
- const item = await env.PB.getWithMetadata(short, { type: "arrayBuffer" })
+ const item = await DB_GetWithMetadata(short, env)
// when paste is not found
- if (item.value === null) {
+ if (safeAccess(item, "value", null) === null) {
throw new WorkerError(404, `paste of name '${short}' not found`)
}
diff --git a/src/handlers/handleWrite.js b/src/handlers/handleWrite.js
index c5138c66..fda43011 100644
--- a/src/handlers/handleWrite.js
+++ b/src/handlers/handleWrite.js
@@ -10,6 +10,7 @@ import {
parsePath,
WorkerError,
} from "../common.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, safeAccess } from "../db.js"
async function createPaste(env, content, isPrivate, expire, short, createDate, passwd, filename) {
const now = new Date().toISOString()
@@ -21,19 +22,19 @@ async function createPaste(env, content, isPrivate, expire, short, createDate, p
if (short === undefined) {
while (true) {
short = genRandStr(short_len)
- if ((await env.PB.get(short)) === null) break
+ if ((await DB_Get(short, env)) === null) break
}
}
- await env.PB.put(short, content, {
+ await DB_Put(short, content, {
expirationTtl: expire,
- metadata: {
+ // metadata: {
postedAt: createDate,
passwd: passwd,
filename: filename,
lastModified: now,
- },
- })
+ // },
+ }, env)
let accessUrl = env.BASE_URL + "/" + short
const adminUrl = env.BASE_URL + "/" + short + params.SEP + passwd
return {
@@ -104,12 +105,14 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
if (isNaN(expirationSeconds)) {
throw new WorkerError(400, `cannot parse expire ${expirationSeconds} as an number`)
}
+ /** KV Only limitation
if (expirationSeconds < 60) {
throw new WorkerError(
400,
`due to limitation of Cloudflare, expire should be a integer greater than 60, '${expirationSeconds}' given`,
)
}
+ */
}
// check if name is legal
@@ -128,8 +131,8 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
if (isPut) {
const { short, passwd } = parsePath(url.pathname)
- const item = await env.PB.getWithMetadata(short)
- if (item.value === null) {
+ const item = await DB_GetWithMetadata(short, env)
+ if (safeAccess(item, "value", null) === null) {
throw new WorkerError(404, `paste of name '${short}' is not found`)
} else {
const date = item.metadata?.postedAt
@@ -145,7 +148,7 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
let short = undefined
if (name !== undefined) {
short = "~" + name
- if ((await env.PB.get(short)) !== null)
+ if ((await DB_Get(short, env)) !== null)
throw new WorkerError(409, `name '${name}' is already used`)
}
return makeResponse(await createPaste(
diff --git a/src/pages/highlight.js b/src/pages/highlight.js
index edcc1ffd..563759d3 100644
--- a/src/pages/highlight.js
+++ b/src/pages/highlight.js
@@ -8,14 +8,14 @@ export function makeHighlight(content, lang) {
-
-
+
+