Chunked + background uploading

package.json

@@ -16,6 +16,7 @@
     "@headlessui/react": "^2.0.3",
     "@tabler/icons-react": "^3.2.0",
     "axios": "^1.7.2",
+    "dexie": "^4.0.11",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-router": "^6.22.3",

public/sw.js

@@ -0,0 +1,89 @@
+importScripts('https://cdnjs.cloudflare.com/ajax/libs/dexie/4.0.10/dexie.min.js');
+
+class SWDB extends Dexie {
+    constructor() {
+        super('UploadDB');
+        this.version(1).stores({
+            uploads: '++id, challengeId, attemptId, chunkIndex',
+            token: 'id, token'
+        });
+
+        this.uploads = this.table('uploads');
+        this.token = this.table('token');
+    }
+
+    async saveToken(token) {
+        await this.token.put({ id: 1, token });
+    }
+
+    async getToken() {
+        const tokenRecord = await this.token.get(1);
+        return tokenRecord ? tokenRecord.token : null;
+    }
+}
+
+const db = new SWDB();
+
+self.addEventListener('message', (event) => {
+    if (event.data.type === 'SET_TOKEN') {
+        db.saveToken(event.data.token)
+        // const token = event.data.token;
+        // // Now you can store the token in the service worker's context or use it
+        // // Example: Store it in a variable or send it with network requests
+        // self.token = token;
+    }
+});
+
+self.addEventListener('sync', (event) => {
+    if (event.tag === 'sync-uploads') {
+        event.waitUntil(retryFailedUploads());
+    }
+});
+
+// Retry failed uploads using Dexie.js
+async function retryFailedUploads() {
+    try {
+        const authToken = await db.getToken()
+        // const db = new UploadDatabase();  // Instantiate your Dexie database class
+
+        // Fetch all failed uploads from the Dexie DB
+        const uploads = await db.uploads.toArray();
+
+        for (const chunk of uploads) {
+            try {
+                // Recreate FormData voor deze chunk
+                const formData = new FormData();
+                const blob = new Blob([ chunk.data ], { type: chunk.fileType });
+
+                formData.append('chunk', blob);
+                formData.append('fileName', chunk.fileName);
+                formData.append('fileType', chunk.fileType);
+                formData.append('chunkIndex', chunk.chunkIndex);
+
+                // Upload chunk
+                const response = await fetch(`/api/challenges/${chunk.challengeId}/attempt/${chunk.attemptId}`, {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${authToken}`
+                    },
+                    body: formData
+                });
+
+                if (response.ok) {
+                    console.log('Upload successful, removing from IndexedDB');
+                    await db.uploads.delete(chunk.id);  // Delete the entry from Dexie DB after successful upload
+                } else if (response.status === 423) { // currently in review or completed. aka upload no longer needed
+                    console.log('resource is locked, removing from IndexedDB');
+                    await db.uploads.delete(chunk.id);  // Delete the entry from Dexie DB after successful upload
+                } else {
+                    console.error(`Upload failed for challenge ${chunk.challengeId}`);
+                    await db.uploads.delete(chunk.id);
+                }
+            } catch (err) {
+                console.error('Upload retry failed, will retry later', err);
+            }
+        }
+    } catch (error) {
+        console.error('Failed to retrieve uploads from IndexedDB', error);
+    }
+}

src/App.tsx

@@ -9,6 +9,24 @@ const AppRoot = () => {
   const [ visible, setVisible ] = useState(false);
   const [ sidebarHidden, setSidebarHidden ] = useState(true);
 
+    useEffect(() => {
+        if ('serviceWorker' in navigator) {
+            navigator.serviceWorker.register('/sw.js')
+                .then((reg) => console.log('Service Worker Registered', reg))
+                .catch((err) => console.log('Service Worker Registration Failed', err));
+
+            if ('SyncManager' in window) {
+                navigator.serviceWorker.ready.then(async (registration) => {
+                    if ('sync' in registration) {
+                        // https://developer.mozilla.org/en-US/docs/Web/API/SyncManager
+                        const syncManager = registration.sync as { register: (tag: string) => Promise<undefined> };
+                        await syncManager.register('sync-uploads');
+                    }
+                });
+            }
+        }
+    }, []);
+
   useEffect(() => {
     const handleScroll = () => {
       const currentScrollPos = window.scrollY;

src/services/auth.service.tsx

@@ -3,6 +3,9 @@ import { AuthResponse } from "../model/auth.tsx";
 
 export const login = (loginCode: string) => {
   return axios.post<AuthResponse>("/api/authenticate", { username: loginCode, password: loginCode }).then((response) => {
+    if (navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({ type: 'SET_TOKEN', token: response.data.accessToken });
+    }
     localStorage.setItem("token", response.data.accessToken);
     localStorage.setItem("role", response.data.role);
     window.dispatchEvent(new Event('storage'));

src/services/challenge.service.tsx

@@ -1,5 +1,6 @@
-import axios, { AxiosProgressEvent, AxiosRequestConfig } from "axios";
+import axios, { AxiosRequestConfig } from "axios";
 import { Challenge } from "../model/challenge.tsx";
+import { db } from "../utils/DexieDB.ts";
 
 export const getChallenges = () => {
   const config: AxiosRequestConfig = {
@@ -11,16 +12,115 @@ export const getChallenges = () => {
   return axios.get<Challenge[]>("/api/challenges", config);
 }
 
-export const uploadChallenge = (challenge: Challenge, formData: FormData, setUploadPercentage: (percentage: number) => void) => {
+export const uploadChallenge = async (
+    challenge: Challenge,
+    formData: FormData,
+    setUploadPercentage: (percentage: number) => void
+) => {
   const config: AxiosRequestConfig = {
     headers: {
       'content-type': 'multipart/form-data',
-      Authorization: `Bearer ${localStorage.getItem("token")}`
-    },
-    onUploadProgress: (progressEvent: AxiosProgressEvent) => {
-      setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100)))
+      Authorization: `Bearer ${localStorage.getItem('token')}`
     }
+  };
+
+  const chunkSize = 1024 * 1024; // 1MB
+
+
+  const result = await axios.post<string>(`/api/challenges/${challenge.id}/attempt`,
+    Object.fromEntries((formData.getAll("files") as File[]).map((file: File) => [ file.name, Math.ceil(file.size / chunkSize) ]))
+  , { headers: { Authorization: `Bearer ${localStorage.getItem('token')}` } })
+
+  const attemptId = result.data
+
+  await uploadFilesInChunks(
+      formData,
+      challenge.id,
+      attemptId,
+      config,
+      setUploadPercentage
+  )
+};
+
+async function uploadFilesInChunks(formData: FormData, challengeId: number, attemptId: string, config: AxiosRequestConfig, setUploadPercentage: (percentage: number) => void) {
+  const uploads: Promise<void>[] = []
+  const files = formData.getAll('files') as File[];  // Get all files from the 'files' input
+
+  // Bereken totaal aantal chunks
+  const chunkSize = 1024 * 1024; // 1MB
+  const totalChunks = files.reduce((total, file) => {
+    return total + Math.ceil(file.size / chunkSize);
+  }, 0);
+
+  let completedChunks = 0;
+
+  if (files.length > 0) {
+    files.forEach(file => {
+      const fileName = file.name;
+      const fileType = file.type
+      const chunkSize = 1024 * 1024; // 1MB
+      const amountOfChunks = Math.ceil(file.size / chunkSize);
+
+      let start = 0;
+      let end = chunkSize;
+
+      for (let chunkIndex = 0; chunkIndex < amountOfChunks; chunkIndex++) {
+        const chunk = file.slice(start, end);
+        const upload = uploadChunk(challengeId, attemptId, config, chunk, fileName, fileType, chunkIndex)
+            .then(() => {
+              completedChunks++;
+              const percentage = Math.round((completedChunks / totalChunks) * 100);
+              setUploadPercentage(percentage);
+            });
+        uploads.push(upload);
+
+        start = end;
+        end = Math.min(start + chunkSize, file.size);
+      }
+    });
+  } else {
+    console.error('No files found in FormData');
   }
+  await Promise.all(uploads)
+}
+
+async function uploadChunk(challengeId: number, attemptId: string, config: AxiosRequestConfig, chunk: Blob, fileName: string, fileType: string, chunkIndex: number) {
+  const chunkFormData = new FormData();
+  chunkFormData.append('chunk', chunk);
+  chunkFormData.append('fileName', fileName);  // Send the file name for reference
+  chunkFormData.append('fileType', fileType);  // Send the file name for reference
+  chunkFormData.append('chunkIndex', String(chunkIndex));
+
+  try {
+    const response = await axios.post(`/api/challenges/${challengeId}/attempt/${attemptId}`, chunkFormData, config)
+    return response
+
+  } catch (err) {
+    console.warn('Upload failed, saving to IndexedDB for retry later');
+
+
+    await db.uploads.add({
+      challengeId,
+      attemptId,
+      chunkIndex: String(chunkIndex),
+      fileName,
+      fileType,
+      data: await chunk.arrayBuffer(),
+    });
 
-  return axios.post(`/api/challenges/${challenge.id}`, formData, config);
+    // Check if sync is supported
+    const registration = await navigator.serviceWorker.ready;
+
+    if ('sync' in registration) {
+      // https://developer.mozilla.org/en-US/docs/Web/API/SyncManager
+      const syncManager = registration.sync as { register: (tag: string) => Promise<undefined> };
+      await syncManager.register('sync-uploads');
+      return;
+    } else {
+      console.info('No background uploading detected')
+      throw err;
+    }
+  }
 }
+
+

src/services/picture.service.tsx

@@ -32,7 +32,7 @@ export const uploadPicture = (picture: Picture, formData: FormData, setUploadPer
         },
         onUploadProgress: (progressEvent: AxiosProgressEvent) => {
             setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100)))
-        }
+        },
     }
 
     return axios.post(`/api/pictures/${picture.id}`, formData, config);

src/utils/DexieDB.ts

@@ -0,0 +1,26 @@
+import Dexie from 'dexie';
+
+interface UploadEntry {
+    id?: number;
+    challengeId: number;
+    attemptId: string;
+    chunkIndex: string;
+    fileName: string;
+    fileType: string;
+    data: ArrayBuffer;
+}
+
+class UploadDatabase extends Dexie {
+    uploads: Dexie.Table<UploadEntry, number>;
+
+    constructor() {
+        super('UploadDB');
+        this.version(1).stores({
+            uploads: '++id, challengeId, attemptId, chunkIndex',
+        });
+
+        this.uploads = this.table('uploads');
+    }
+}
+
+export const db = new UploadDatabase();

vite.config.ts

@@ -3,6 +3,8 @@ import react from '@vitejs/plugin-react'
 import { createRoutedSitemap } from "./vite/sitemap.vite";
 import { createSEOTags } from "./vite/seo.vite";
 
+const isProduction = process.env.NODE_ENV === 'production';
+
 // https://vitejs.dev/config/
 export default defineConfig({
   plugins: [
@@ -18,7 +20,7 @@ export default defineConfig({
   },
   server: {
     proxy: {
-      '/api': 'https://intro.svindicium.nl'
+      '/api': isProduction ? 'https://intro.svindicium.nl' : 'http://localhost:3080'
     }
   }
 })