feat: support d1 and combine with kv

README.md

@@ -25,22 +25,26 @@ This is a pastebin that can be deployed on Cloudflare workers. Try it on [shz.al
 
 ## Limitations
 
-1. If deployed on Cloudflare Worker free-tier plan, the service allows at most 100,000 reads and 1000 writes, 1000 deletes per day. 
-2. Due to the size limit of Cloudflare KV storage, the size of each paste is bounded under 25 MB. 
+1. If deployed on Cloudflare Worker free-tier plan, the service allows at most 5 million reads and 100000 writes per day.
+2. Due to the size limit of Cloudflare D1 storage, the size of each basic paste is bounded under 1 MB.
+3. Big files (more than 0.95 MB) are storaged in KV (metadata in D1), the size limit is 25 MB. 100000 reads per day. 1000 writes per day.
+4. To save KV writes, we do not actively delete expired big files in KV, we just overwrite them in need.
 
 ## Deploy
 
 You are free to deploy the pastebin on your own domain if you host your domain on Cloudflare. 
 
 1. Install `node` and `yarn`.
 
-2. Create a KV namespace on Cloudflare workers dashboard, remember its ID.
+2. Create a D1 Database on Cloudflare workers dashboard, run `schema.sql` in it, remember its ID. Please notice that executing `schema.sql` in the DB will reset all data in the DB.
 
-3. Clone the repository and enter the directory. Login to your Cloudflare account with `wrangler login`.
+3. Create a KV on Cloudflare workers dashboard, remember its ID.
 
-4. Modify entries in `wrangler.toml`. Its comments will tell you how.
+4. Clone the repository and enter the directory. Login to your Cloudflare account with `wrangler login`.
 
-5. Deploy and enjoy!
+5. Modify entries in `wrangler.toml`. Its comments will tell you how.
+
+6. Deploy and enjoy!
 
 ```shell
 $ yarn install

frontend/index.html

@@ -43,7 +43,7 @@ <h2>Settings</h2>
         <div id='paste-expiration-panel' class='paste-setting-subitem-panel'>
             <input list='expiration-choices' type='text' min='60' step='1' name='paste-expiration'
                    id='paste-expiration-input' placeholder='Expiration (secs)' value='7d'>
-            <label class='small-label' for='paste-expiration-input'>Delete your paste after a period of time. <br>Units: s (seconds), m (minutes), h (hours), d (days), M (months)</label>
+            <label class='small-label' for='paste-expiration-input'>Delete your paste after a period of time, 0 for no expiration. <br>Available units: default (seconds), m (minutes), h (hours), d (days), M (months)</label>
         </div>
         <div id='paste-passwd-panel' class='paste-setting-subitem-panel'>
             <input type='text' spellcheck='false' name='paste-expiration' id='paste-passwd-input' placeholder='Password'>

schema.sql

@@ -0,0 +1,13 @@
+-- PB schema
+-- In the old KV design, they use metadata to store password, date, etc.
+-- But there's no such metadata in SQLite (D1).
+-- So we create a new table for them.
+-- short: the short url (string)
+-- content: the paste or file content (blob)
+-- metadata: JSON string of metadata (string)
+DROP TABLE IF EXISTS pastes;
+CREATE TABLE IF NOT EXISTS pastes (
+    short TEXT PRIMARY KEY,
+    content BLOB,
+    metadata TEXT
+)

src/db.js

@@ -0,0 +1,166 @@
+// Cloudflare D1 SQLite DB and KV for big file storage
+import { WorkerError } from "./common.js"
+
+/**
+  await env.PB.put(short, content, {
+    expirationTtl: expire,
+    postedAt: createDate,
+    passwd: passwd,
+    filename: filename,
+    lastModified: now,
+  })
+*/
+
+export function safeAccess(obj, prop, defaultValue) {
+  try {
+    if (obj && typeof obj === "object" && prop in obj) {
+      return obj[prop]
+    } else {
+      return defaultValue
+    }
+  } catch (e) {
+    return defaultValue
+  }
+}
+
+const expiredItemTemplate = {
+  value: "Expired paste",
+  metadata: {
+    expirationTtl: 0,
+    postedAt: Date.toString(Date.now() / 1000 + 10), // let Client Cache reset in 10 seconds
+    passwd: "",
+    filename: "",
+    lastModified: Date.toString(Date.now() / 1000 + 10),
+  },
+}
+
+const NotExistItemTemplate = {
+  value: "Not found",
+  metadata: {
+    expirationTtl: 0,
+    postedAt: Date.toString(Date.now() / 1000),
+    passwd: "",
+    filename: "",
+    lastModified: Date.toString(Date.now() / 1000),
+  },
+}
+
+function isExpired(item, env) {
+  // return true if item is expired
+  const realMetadata = JSON.parse(item.metadata)
+  // Use postedAt, or use lastModified is existing
+  if (realMetadata.lastModified === undefined) {
+    var lastModified = realMetadata.postedAt
+  } else {
+    var lastModified = realMetadata.lastModified
+  }
+  const lastModifiedUnix = Date.parse(lastModified) / 1000 // In seconds
+  const shouldExpireTime = lastModifiedUnix + realMetadata.expirationTtl
+
+  // if realMetadata.expirationTtl is 0, it should never expire
+  if (realMetadata.expirationTtl === 0) return false
+
+  const nowUnix = Date.now() / 1000
+  return nowUnix > shouldExpireTime
+}
+
+async function KV_Put(short, content, env) {
+  await env.KV.put(short, content)
+}
+
+async function KV_Get(short, env) {
+  const content = await env.KV.get(short, { type: "arrayBuffer" })
+  return new Uint8Array(content)
+}
+
+export async function DB_Put(short, content, metadata, env) {
+  // If content is bigger than 0.95MB, save to KV
+  if (content.length > 1024 * 1024 * 0.95) {
+    await KV_Put(short, content, env)
+    content = "{KV_Storaged_Flag_Attention_DO_NOT_DELETE_OR_MODIFY_THIS_LINE}"
+  }
+
+  return await env.DB.prepare(
+    "INSERT OR REPLACE INTO pastes (short, content, metadata) VALUES (?, ?, ?)",
+  )
+    .bind(short, content, JSON.stringify(metadata))
+    .run() // run and don't return actual result
+}
+
+export async function DB_Get(short, env) {
+  const item_db = await env.DB.prepare("SELECT * FROM pastes WHERE short = ?")
+    .bind(short)
+    .first() // run and return first result
+
+  // Check existence
+  if (!item_db) {
+    return null
+  }
+
+  if (isExpired(item_db, env)) {
+    await DB_Delete(short, env)
+    return null
+  }
+
+  return new Uint8Array(item_db.content)
+}
+
+/**
+    const item = await env.PB.getWithMetadata(short)
+    if (item.value === null) {
+      throw new WorkerError(404, `paste of name '${short}' is not found`)
+    } else {
+      const date = item.metadata?.postedAt
+      if (passwd !== item.metadata?.passwd) {
+        throw new WorkerError(403, `incorrect password for paste '${short}`)
+      } else {
+        return makeResponse(
+          await createPaste(env, content, isPrivate, expirationSeconds, short, date, newPasswd || passwd, filename),
+        )
+      }
+    }
+*/
+
+export async function DB_GetWithMetadata(short, env) {
+  const item_db = await env.DB.prepare("SELECT * FROM pastes WHERE short = ?")
+    .bind(short)
+    .first()
+
+  // Check existence
+  if (!item_db) {
+    // return NotExistItemTemplate
+    return null
+  }
+
+  if (isExpired(item_db, env)) {
+    await DB_Delete(short, env)
+    return expiredItemTemplate // This function is not used to check if the paste exists
+    // So it's okay to return non-null expiredItemTemplate
+  }
+
+  if (
+    item_db.content ===
+    "{KV_Storaged_Flag_Attention_DO_NOT_DELETE_OR_MODIFY_THIS_LINE}"
+  ) {
+    const kv_content = await KV_Get(short, env)
+    var item = {
+      value: kv_content,
+      metadata: JSON.parse(item_db.metadata),
+    }
+  } else {
+    var item = {
+      value: new Uint8Array(item_db.content),
+      metadata: JSON.parse(item_db.metadata),
+    }
+  }
+
+  // console.log(item)
+
+  return item
+}
+
+export async function DB_Delete(short, env) {
+  return await env.DB.prepare("DELETE FROM pastes WHERE short = ?")
+    .bind(short)
+    .run() // Run and don't return actual result
+}

src/handlers/handleDelete.js

@@ -1,16 +1,17 @@
 import { parsePath, WorkerError } from "../common.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, DB_Delete, safeAccess } from "../db.js"
 
 export async function handleDelete(request, env, ctx) {
   const url = new URL(request.url)
   const { short, passwd } = parsePath(url.pathname)
-  const item = await env.PB.getWithMetadata(short)
-  if (item.value === null) {
+  const item = await DB_GetWithMetadata(short, env)
+  if (safeAccess(item, "value", null) === null) {
     throw new WorkerError(404, `paste of name '${short}' not found`)
   } else {
     if (passwd !== item.metadata?.passwd) {
       throw new WorkerError(403, `incorrect password for paste '${short}`)
     } else {
-      await env.PB.delete(short)
+      await DB_Delete(short, env)
       return new Response("the paste will be deleted in seconds")
     }
   }

src/handlers/handleRead.js

@@ -4,6 +4,7 @@ import { verifyAuth } from "../auth.js"
 import { getType } from "mime/lite.js"
 import { makeMarkdown } from "../pages/markdown.js"
 import { makeHighlight } from "../pages/highlight.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, safeAccess } from "../db.js"
 
 function staticPageCacheHeader(env) {
   const age = env.CACHE_STATIC_PAGE_AGE
@@ -45,10 +46,10 @@ export async function handleGet(request, env, ctx) {
 
   const disp = url.searchParams.has("a") ? "attachment" : "inline"
 
-  const item = await env.PB.getWithMetadata(short, { type: "arrayBuffer" })
+  const item = await DB_GetWithMetadata(short, env)
 
   // when paste is not found
-  if (item.value === null) {
+  if (safeAccess(item, "value", null) === null) {
     throw new WorkerError(404, `paste of name '${short}' not found`)
   }
 

src/handlers/handleWrite.js

@@ -10,6 +10,7 @@ import {
   parsePath,
   WorkerError,
 } from "../common.js"
+import { DB_Put, DB_Get, DB_GetWithMetadata, safeAccess } from "../db.js"
 
 async function createPaste(env, content, isPrivate, expire, short, createDate, passwd, filename) {
   const now = new Date().toISOString()
@@ -21,19 +22,19 @@ async function createPaste(env, content, isPrivate, expire, short, createDate, p
   if (short === undefined) {
     while (true) {
       short = genRandStr(short_len)
-      if ((await env.PB.get(short)) === null) break
+      if ((await DB_Get(short, env)) === null) break
     }
   }
 
-  await env.PB.put(short, content, {
+  await DB_Put(short, content, {
     expirationTtl: expire,
-    metadata: {
+    // metadata: {
       postedAt: createDate,
       passwd: passwd,
       filename: filename,
       lastModified: now,
-    },
-  })
+    // },
+  }, env)
   let accessUrl = env.BASE_URL + "/" + short
   const adminUrl = env.BASE_URL + "/" + short + params.SEP + passwd
   return {
@@ -104,12 +105,14 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
     if (isNaN(expirationSeconds)) {
       throw new WorkerError(400, `cannot parse expire ${expirationSeconds} as an number`)
     }
+    /** KV Only limitation
     if (expirationSeconds < 60) {
       throw new WorkerError(
         400,
         `due to limitation of Cloudflare, expire should be a integer greater than 60, '${expirationSeconds}' given`,
       )
     }
+    */
   }
 
   // check if name is legal
@@ -128,8 +131,8 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
 
   if (isPut) {
     const { short, passwd } = parsePath(url.pathname)
-    const item = await env.PB.getWithMetadata(short)
-    if (item.value === null) {
+    const item = await DB_GetWithMetadata(short, env)
+    if (safeAccess(item, "value", null) === null) {
       throw new WorkerError(404, `paste of name '${short}' is not found`)
     } else {
       const date = item.metadata?.postedAt
@@ -145,7 +148,7 @@ export async function handlePostOrPut(request, env, ctx, isPut) {
     let short = undefined
     if (name !== undefined) {
       short = "~" + name
-      if ((await env.PB.get(short)) !== null)
+      if ((await DB_Get(short, env)) !== null)
         throw new WorkerError(409, `name '${name}' is already used`)
     }
     return makeResponse(await createPaste(

src/pages/highlight.js

@@ -8,14 +8,14 @@ export function makeHighlight(content, lang) {
   <meta charset='utf-8'>
   <meta name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no'>
   <meta charset='utf-8'>
-  <link href='https://cdn.jsdelivr.net/npm/[email protected]/themes/prism.css' rel='stylesheet' />
-  <link href='https://cdn.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.css' rel='stylesheet' />
+  <link href='https://fastly.jsdelivr.net/npm/[email protected]/themes/prism.css' rel='stylesheet' />
+  <link href='https://fastly.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.css' rel='stylesheet' />
 </head>
 <body class='line-numbers'>
 <pre><code class='language-${escapeHtml(lang)}'>${escapeHtml(content)}</code></pre>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/components/prism-core.min.js'></script>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.min.js'></script>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/plugins/autoloader/prism-autoloader.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/components/prism-core.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/plugins/autoloader/prism-autoloader.min.js'></script>
 </body>
 </html>
 `

src/pages/markdown.js

@@ -72,21 +72,21 @@ export function makeMarkdown(content) {
   <meta name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no'>
   <title>${metadata.title}</title>
   ${metadata.description.length > 0 ? `<meta name='description' content='${metadata.description}'>` : ""}
-  <link href='https://cdn.jsdelivr.net/npm/[email protected]/themes/prism.css' rel='stylesheet' />
-  <link href='https://cdn.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.css' rel='stylesheet' />
+  <link href='https://fastly.jsdelivr.net/npm/[email protected]/themes/prism.css' rel='stylesheet' />
+  <link href='https://fastly.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.css' rel='stylesheet' />
   <link rel='stylesheet' href='https://pages.github.com/assets/css/style.css'>
   <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
   <script id="MathJax-script" async
-          src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js">
+          src="https://fastly.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js">
   </script>
 </head>
 <body>
 <article class='line-numbers container-lg px-3 my-5 markdown-body'>
 ${convertedHtml}
 </article>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/components/prism-core.min.js'></script>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.min.js'></script>
-  <script src='https://cdn.jsdelivr.net/npm/[email protected]/plugins/autoloader/prism-autoloader.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/components/prism-core.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/plugins/line-numbers/prism-line-numbers.min.js'></script>
+  <script src='https://fastly.jsdelivr.net/npm/[email protected]/plugins/autoloader/prism-autoloader.min.js'></script>
 </html>
 `
 }