From a0a0da82b23dccf0ccb288a6003b70f81909f670 Mon Sep 17 00:00:00 2001 From: kaasbroodju Date: Tue, 25 Feb 2025 17:42:29 +0100 Subject: [PATCH 1/4] Add: background sync --- package-lock.json | 27 +++++++-- package.json | 1 + public/sw.js | 92 ++++++++++++++++++++++++++++++ src/App.tsx | 14 +++++ src/services/auth.service.tsx | 3 + src/services/challenge.service.tsx | 54 ++++++++++++++++-- src/services/picture.service.tsx | 2 +- src/utils/DexieDB.ts | 23 ++++++++ vite.config.ts | 4 +- 9 files changed, 207 insertions(+), 13 deletions(-) create mode 100644 public/sw.js create mode 100644 src/utils/DexieDB.ts diff --git a/package-lock.json b/package-lock.json index ef9966a..5d5fdf4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@headlessui/react": "^2.0.3", "@tabler/icons-react": "^3.2.0", "axios": "^1.7.2", + "dexie": "^4.0.11", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router": "^6.22.3", @@ -2209,6 +2210,12 @@ "node": ">=0.4.0" } }, + "node_modules/dexie": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-4.0.11.tgz", + "integrity": "sha512-SOKO002EqlvBYYKQSew3iymBoN2EQ4BDw/3yprjh7kAfFzjBYkaMNa/pZvcA7HSWlcKSQb9XhPe3wKyQ0x4A8A==", + "license": "Apache-2.0" + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -5377,7 +5384,8 @@ "@react-types/shared": { "version": "3.23.0", "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.23.0.tgz", - "integrity": "sha512-GQm/iPiii3ikcaMNR4WdVkJ4w0mKtV3mLqeSfSqzdqbPr6vONkqXbh3RhPlPmAJs1b4QHnexd/wZQP3U9DHOwQ==" + "integrity": "sha512-GQm/iPiii3ikcaMNR4WdVkJ4w0mKtV3mLqeSfSqzdqbPr6vONkqXbh3RhPlPmAJs1b4QHnexd/wZQP3U9DHOwQ==", + "requires": {} }, "@remix-run/router": { "version": "1.15.3", @@ -5744,7 +5752,8 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true + "dev": true, + "requires": {} }, "ajv": { "version": "6.12.6", @@ -6023,6 +6032,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "dexie": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-4.0.11.tgz", + "integrity": "sha512-SOKO002EqlvBYYKQSew3iymBoN2EQ4BDw/3yprjh7kAfFzjBYkaMNa/pZvcA7HSWlcKSQb9XhPe3wKyQ0x4A8A==" + }, "didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -6249,13 +6263,15 @@ "version": "4.6.0", "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz", "integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==", - "dev": true + "dev": true, + "requires": {} }, "eslint-plugin-react-refresh": { "version": "0.4.6", "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.6.tgz", "integrity": "sha512-NjGXdm7zgcKRkKMua34qVO9doI7VOxZ6ancSvBELJSSoX97jyndXcSoa8XBh69JoB31dNz3EEzlMcizZl7LaMA==", - "dev": true + "dev": true, + "requires": {} }, "eslint-scope": { "version": "7.2.2", @@ -7477,7 +7493,8 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", - "dev": true + "dev": true, + "requires": {} }, "ts-interface-checker": { "version": "0.1.13", diff --git a/package.json b/package.json index 2531606..dd475f6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@headlessui/react": "^2.0.3", "@tabler/icons-react": "^3.2.0", "axios": "^1.7.2", + "dexie": "^4.0.11", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router": "^6.22.3", diff --git a/public/sw.js b/public/sw.js new file mode 100644 index 0000000..6918b9a --- /dev/null +++ b/public/sw.js @@ -0,0 +1,92 @@ +importScripts('https://cdnjs.cloudflare.com/ajax/libs/dexie/4.0.10/dexie.min.js'); + +class SWDB extends Dexie { + constructor() { + super('UploadDB'); + this.version(1).stores({ + uploads: '++id, challengeId', + token: 'id, token' + }); + + this.uploads = this.table('uploads'); + this.token = this.table('token'); + } + + async saveToken(token) { + await this.token.put({ id: 1, token }); + } + + async getToken() { + const tokenRecord = await this.token.get(1); + return tokenRecord ? tokenRecord.token : null; + } +} + +const db = new SWDB(); + +self.addEventListener('message', (event) => { + if (event.data.type === 'SET_TOKEN') { + db.saveToken(event.data.token) + // const token = event.data.token; + // // Now you can store the token in the service worker's context or use it + // // Example: Store it in a variable or send it with network requests + // self.token = token; + } +}); + +self.addEventListener('sync', (event) => { + if (event.tag === 'sync-uploads') { + event.waitUntil(retryFailedUploads()); + } +}); + +// Retry failed uploads using Dexie.js +async function retryFailedUploads() { + try { + const authToken = await db.getToken() + // const db = new UploadDatabase(); // Instantiate your Dexie database class + + // Fetch all failed uploads from the Dexie DB + const uploads = await db.uploads.toArray(); + + for (const entry of uploads) { + const formData = new FormData(); + + // Convert ArrayBuffer back to Blob and append each file + for (const file of entry.files) { + const blob = new Blob([file.fileData], { type: file.fileType }); + formData.append('files', blob, file.fileName); + } + + // Append other form fields to FormData + for (const [key, value] of Object.entries(entry.additionalData)) { + formData.append(key, value); + } + + try { + // Attempt to upload the files again + const response = await fetch(`/api/challenges/${entry.challengeId}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${authToken}` + }, + body: formData + }); + + if (response.ok) { + console.log('Upload successful, removing from IndexedDB'); + await db.uploads.delete(entry.id); // Delete the entry from Dexie DB after successful upload + } else if (response.status === 423) { // currently in review or completed. aka upload no longer needed + console.log('resource is locked, removing from IndexedDB'); + await db.uploads.delete(entry.id); // Delete the entry from Dexie DB after successful upload + } else { + console.error(`Upload failed for challenge ${entry.challengeId}`); + } + } catch (err) { + console.error('Upload retry failed, will retry later', err); + } + } + } catch (error) { + console.error('Failed to retrieve uploads from IndexedDB', error); + } +} diff --git a/src/App.tsx b/src/App.tsx index e64a355..db8e2fd 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,6 +9,20 @@ const AppRoot = () => { const [ visible, setVisible ] = useState(false); const [ sidebarHidden, setSidebarHidden ] = useState(true); + useEffect(() => { + if ('serviceWorker' in navigator) { + navigator.serviceWorker.register('/sw.js') + .then((reg) => console.log('Service Worker Registered', reg)) + .catch((err) => console.log('Service Worker Registration Failed', err)); + + if ('SyncManager' in window) { + navigator.serviceWorker.ready.then((reg) => { + reg.sync.register('sync-uploads'); + }); + } + } + }, []); + useEffect(() => { const handleScroll = () => { const currentScrollPos = window.scrollY; diff --git a/src/services/auth.service.tsx b/src/services/auth.service.tsx index ae684f9..6bb9e3f 100644 --- a/src/services/auth.service.tsx +++ b/src/services/auth.service.tsx @@ -3,6 +3,9 @@ import { AuthResponse } from "../model/auth.tsx"; export const login = (loginCode: string) => { return axios.post("/api/authenticate", { username: loginCode, password: loginCode }).then((response) => { + if (navigator.serviceWorker.controller) { + navigator.serviceWorker.controller.postMessage({ type: 'SET_TOKEN', token: response.data.accessToken }); + } localStorage.setItem("token", response.data.accessToken); localStorage.setItem("role", response.data.role); window.dispatchEvent(new Event('storage')); diff --git a/src/services/challenge.service.tsx b/src/services/challenge.service.tsx index 97c9ce4..f678771 100644 --- a/src/services/challenge.service.tsx +++ b/src/services/challenge.service.tsx @@ -1,5 +1,6 @@ import axios, { AxiosProgressEvent, AxiosRequestConfig } from "axios"; import { Challenge } from "../model/challenge.tsx"; +import { db } from "../utils/DexieDB.ts"; export const getChallenges = () => { const config: AxiosRequestConfig = { @@ -11,16 +12,57 @@ export const getChallenges = () => { return axios.get("/api/challenges", config); } -export const uploadChallenge = (challenge: Challenge, formData: FormData, setUploadPercentage: (percentage: number) => void) => { +export const uploadChallenge = async ( + challenge: Challenge, + formData: FormData, + setUploadPercentage: (percentage: number) => void +) => { const config: AxiosRequestConfig = { headers: { 'content-type': 'multipart/form-data', - Authorization: `Bearer ${localStorage.getItem("token")}` + Authorization: `Bearer ${localStorage.getItem('token')}` }, onUploadProgress: (progressEvent: AxiosProgressEvent) => { - setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100))) + setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100))); } - } + }; - return axios.post(`/api/challenges/${challenge.id}`, formData, config); -} + try { + const response = await axios.post(`/api/challenges/${challenge.id}`, formData, config); + return response; + } catch (error) { + console.warn('Upload failed, saving to IndexedDB for retry later'); + + // Extract multiple files and additional fields + const filesArray: { fileName: string; fileType: string; fileData: ArrayBuffer }[] = []; + const additionalData: { [key: string]: string } = {}; + + const files = formData.getAll('files') as File[]; + for (const file of files) { + const fileData = await file.arrayBuffer(); + filesArray.push({ fileName: file.name, fileType: file.type, fileData }); + } + + formData.forEach((value, key) => { + if (key !== 'files') additionalData[key] = value as string; + }); + + await db.uploads.add({ + challengeId: challenge.id, + files: filesArray, + additionalData + }); + + // Check if sync is supported + + const registration = await navigator.serviceWorker.ready; + + if ('sync' in registration) { + registration.sync.register('sync-uploads'); + } else { + console.info('No background uploading detected') + } + + throw error; + } +}; diff --git a/src/services/picture.service.tsx b/src/services/picture.service.tsx index 21b9947..e022071 100644 --- a/src/services/picture.service.tsx +++ b/src/services/picture.service.tsx @@ -32,7 +32,7 @@ export const uploadPicture = (picture: Picture, formData: FormData, setUploadPer }, onUploadProgress: (progressEvent: AxiosProgressEvent) => { setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100))) - } + }, } return axios.post(`/api/pictures/${picture.id}`, formData, config); diff --git a/src/utils/DexieDB.ts b/src/utils/DexieDB.ts new file mode 100644 index 0000000..386290f --- /dev/null +++ b/src/utils/DexieDB.ts @@ -0,0 +1,23 @@ +import Dexie from 'dexie'; + +interface UploadEntry { + id?: number; + challengeId: string; + files: { fileName: string; fileType: string; fileData: ArrayBuffer }[]; // Store multiple files + additionalData: { [key: string]: string }; // Store other FormData fields +} + +class UploadDatabase extends Dexie { + uploads: Dexie.Table; + + constructor() { + super('UploadDB'); + this.version(1).stores({ + uploads: '++id, challengeId' + }); + + this.uploads = this.table('uploads'); + } +} + +export const db = new UploadDatabase(); diff --git a/vite.config.ts b/vite.config.ts index 6971eac..8c860f9 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -3,6 +3,8 @@ import react from '@vitejs/plugin-react' import { createRoutedSitemap } from "./vite/sitemap.vite"; import { createSEOTags } from "./vite/seo.vite"; +const isProduction = process.env.NODE_ENV === 'production'; + // https://vitejs.dev/config/ export default defineConfig({ plugins: [ @@ -18,7 +20,7 @@ export default defineConfig({ }, server: { proxy: { - '/api': 'https://intro.svindicium.nl' + '/api': isProduction ? 'https://intro.svindicium.nl' : 'http://localhost:3080' } } }) From 327a27b76aeb034aee9a254b56f26f5303450cfc Mon Sep 17 00:00:00 2001 From: kaasbroodju Date: Wed, 26 Feb 2025 23:03:52 +0100 Subject: [PATCH 2/4] Refactor: chunked uploading --- src/services/challenge.service.tsx | 132 +++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 33 deletions(-) diff --git a/src/services/challenge.service.tsx b/src/services/challenge.service.tsx index f678771..4ea018f 100644 --- a/src/services/challenge.service.tsx +++ b/src/services/challenge.service.tsx @@ -1,4 +1,4 @@ -import axios, { AxiosProgressEvent, AxiosRequestConfig } from "axios"; +import axios, { AxiosProgressEvent, AxiosRequestConfig, AxiosResponse } from "axios"; import { Challenge } from "../model/challenge.tsx"; import { db } from "../utils/DexieDB.ts"; @@ -27,42 +27,108 @@ export const uploadChallenge = async ( } }; - try { - const response = await axios.post(`/api/challenges/${challenge.id}`, formData, config); - return response; - } catch (error) { - console.warn('Upload failed, saving to IndexedDB for retry later'); - - // Extract multiple files and additional fields - const filesArray: { fileName: string; fileType: string; fileData: ArrayBuffer }[] = []; - const additionalData: { [key: string]: string } = {}; - - const files = formData.getAll('files') as File[]; - for (const file of files) { - const fileData = await file.arrayBuffer(); - filesArray.push({ fileName: file.name, fileType: file.type, fileData }); - } + const chunkSize = 1024 * 1024; // 1MB + - formData.forEach((value, key) => { - if (key !== 'files') additionalData[key] = value as string; - }); + const result = await axios.post(`/api/challenges/${challenge.id}/attempt`, + Object.fromEntries((formData.getAll("files") as File[]).map((file: File) => [ file.name, Math.ceil(file.size / chunkSize) ])) + , { headers: { Authorization: `Bearer ${localStorage.getItem('token')}` } }) - await db.uploads.add({ - challengeId: challenge.id, - files: filesArray, - additionalData - }); + const attemptId = result.data + console.log(result.data); + + uploadFilesInChunks(formData, (temp) => axios.post(`/api/challenges/${challenge.id}/attempt/${attemptId}`, temp, config)) - // Check if sync is supported + // try { + // const response = await axios.post(`/api/challenges/${challenge.id}`, formData, config); + // return response; + // } catch (error) { + // console.warn('Upload failed, saving to IndexedDB for retry later'); + // + // // Extract multiple files and additional fields + // const filesArray: { fileName: string; fileType: string; fileData: ArrayBuffer }[] = []; + // const additionalData: { [key: string]: string } = {}; + // + // const files = formData.getAll('files') as File[]; + // for (const file of files) { + // const fileData = await file.arrayBuffer(); + // filesArray.push({ fileName: file.name, fileType: file.type, fileData }); + // } + // + // formData.forEach((value, key) => { + // if (key !== 'files') additionalData[key] = value as string; + // }); + // + // await db.uploads.add({ + // challengeId: challenge.id, + // files: filesArray, + // additionalData + // }); + // + // // Check if sync is supported + // + // const registration = await navigator.serviceWorker.ready; + // + // if ('sync' in registration) { + // registration.sync.register('sync-uploads'); + // } else { + // console.info('No background uploading detected') + // } + // + // throw error; + // } +}; - const registration = await navigator.serviceWorker.ready; +function uploadFilesInChunks(formData: FormData, uploadMethod: (data: FormData) => Promise>) { + const files = formData.getAll('files') as File[]; // Get all files from the 'files' input - if ('sync' in registration) { - registration.sync.register('sync-uploads'); - } else { - console.info('No background uploading detected') - } + if (files.length > 0) { + files.forEach(file => { + const fileName = file.name; + const fileType = file.type + const chunkSize = 1024 * 1024; // 1MB + const totalChunks = Math.ceil(file.size / chunkSize); + + let start = 0; + let end = chunkSize; + + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const chunk = file.slice(start, end); + uploadChunk(chunk, fileName, fileType, chunkIndex, uploadMethod); + + start = end; + end = Math.min(start + chunkSize, file.size); + } + }); + } else { + console.error('No files found in FormData'); + } +} + +async function uploadChunk(chunk: Blob, fileName: string, fileType: string, chunkIndex: number, uploadMethod: (data: FormData) => Promise>) { + const chunkFormData = new FormData(); + chunkFormData.append('chunk', chunk); + chunkFormData.append('fileName', fileName); // Send the file name for reference + chunkFormData.append('fileType', fileType); // Send the file name for reference + chunkFormData.append('chunkIndex', String(chunkIndex)); + + try { + const response = await uploadMethod(chunkFormData) + return response - throw error; + } catch (err) { + console.error(err) } -}; + // uploadMethod(chunkFormData) + // .then(response => { + // if (!response.ok) { + // throw new Error('Chunk upload failed'); + // } + // console.log(`Chunk ${chunkIndex + 1} of ${totalChunks} for ${fileName} uploaded successfully.`); + // }) + // .catch(error => { + // console.error('Error uploading chunk:', error); + // }); +} + + From 217279673b00a9751076fe47f97e8c1ef799f976 Mon Sep 17 00:00:00 2001 From: kaasbroodju Date: Tue, 22 Jul 2025 01:22:54 +0200 Subject: [PATCH 3/4] Refactor: chunked uploading now uses background sync Fix: correct percentage of uploading is now being shown --- public/sw.js | 35 ++++----- src/services/challenge.service.tsx | 115 +++++++++++++---------------- src/utils/DexieDB.ts | 11 ++- 3 files changed, 76 insertions(+), 85 deletions(-) diff --git a/public/sw.js b/public/sw.js index 6918b9a..f9169b5 100644 --- a/public/sw.js +++ b/public/sw.js @@ -4,7 +4,7 @@ class SWDB extends Dexie { constructor() { super('UploadDB'); this.version(1).stores({ - uploads: '++id, challengeId', + uploads: '++id, challengeId, attemptId, chunkIndex', token: 'id, token' }); @@ -49,23 +49,19 @@ async function retryFailedUploads() { // Fetch all failed uploads from the Dexie DB const uploads = await db.uploads.toArray(); - for (const entry of uploads) { - const formData = new FormData(); + for (const chunk of uploads) { + try { + // Recreate FormData voor deze chunk + const formData = new FormData(); + const blob = new Blob([ chunk.data ], { type: chunk.fileType }); - // Convert ArrayBuffer back to Blob and append each file - for (const file of entry.files) { - const blob = new Blob([file.fileData], { type: file.fileType }); - formData.append('files', blob, file.fileName); - } + formData.append('chunk', blob); + formData.append('fileName', chunk.fileName); + formData.append('fileType', chunk.fileType); + formData.append('chunkIndex', chunk.chunkIndex); - // Append other form fields to FormData - for (const [key, value] of Object.entries(entry.additionalData)) { - formData.append(key, value); - } - - try { - // Attempt to upload the files again - const response = await fetch(`/api/challenges/${entry.challengeId}`, { + // Upload chunk + const response = await fetch(`/api/challenges/${chunk.challengeId}/attempt/${chunk.attemptId}`, { method: 'POST', headers: { Authorization: `Bearer ${authToken}` @@ -75,12 +71,13 @@ async function retryFailedUploads() { if (response.ok) { console.log('Upload successful, removing from IndexedDB'); - await db.uploads.delete(entry.id); // Delete the entry from Dexie DB after successful upload + await db.uploads.delete(chunk.id); // Delete the entry from Dexie DB after successful upload } else if (response.status === 423) { // currently in review or completed. aka upload no longer needed console.log('resource is locked, removing from IndexedDB'); - await db.uploads.delete(entry.id); // Delete the entry from Dexie DB after successful upload + await db.uploads.delete(chunk.id); // Delete the entry from Dexie DB after successful upload } else { - console.error(`Upload failed for challenge ${entry.challengeId}`); + console.error(`Upload failed for challenge ${chunk.challengeId}`); + await db.uploads.delete(chunk.id); } } catch (err) { console.error('Upload retry failed, will retry later', err); diff --git a/src/services/challenge.service.tsx b/src/services/challenge.service.tsx index 4ea018f..6a6b54f 100644 --- a/src/services/challenge.service.tsx +++ b/src/services/challenge.service.tsx @@ -21,9 +21,6 @@ export const uploadChallenge = async ( headers: { 'content-type': 'multipart/form-data', Authorization: `Bearer ${localStorage.getItem('token')}` - }, - onUploadProgress: (progressEvent: AxiosProgressEvent) => { - setUploadPercentage(Math.round((progressEvent.loaded / (progressEvent.total ?? 1) * 100))); } }; @@ -35,66 +32,47 @@ export const uploadChallenge = async ( , { headers: { Authorization: `Bearer ${localStorage.getItem('token')}` } }) const attemptId = result.data - console.log(result.data); - - uploadFilesInChunks(formData, (temp) => axios.post(`/api/challenges/${challenge.id}/attempt/${attemptId}`, temp, config)) - - // try { - // const response = await axios.post(`/api/challenges/${challenge.id}`, formData, config); - // return response; - // } catch (error) { - // console.warn('Upload failed, saving to IndexedDB for retry later'); - // - // // Extract multiple files and additional fields - // const filesArray: { fileName: string; fileType: string; fileData: ArrayBuffer }[] = []; - // const additionalData: { [key: string]: string } = {}; - // - // const files = formData.getAll('files') as File[]; - // for (const file of files) { - // const fileData = await file.arrayBuffer(); - // filesArray.push({ fileName: file.name, fileType: file.type, fileData }); - // } - // - // formData.forEach((value, key) => { - // if (key !== 'files') additionalData[key] = value as string; - // }); - // - // await db.uploads.add({ - // challengeId: challenge.id, - // files: filesArray, - // additionalData - // }); - // - // // Check if sync is supported - // - // const registration = await navigator.serviceWorker.ready; - // - // if ('sync' in registration) { - // registration.sync.register('sync-uploads'); - // } else { - // console.info('No background uploading detected') - // } - // - // throw error; - // } + + await uploadFilesInChunks( + formData, + challenge.id, + attemptId, + config, + setUploadPercentage + ) }; -function uploadFilesInChunks(formData: FormData, uploadMethod: (data: FormData) => Promise>) { +async function uploadFilesInChunks(formData: FormData, challengeId: number, attemptId: any, config: AxiosRequestConfig, setUploadPercentage: (percentage: number) => void) { + const uploads: Promise[] = [] const files = formData.getAll('files') as File[]; // Get all files from the 'files' input + // Bereken totaal aantal chunks + const chunkSize = 1024 * 1024; // 1MB + const totalChunks = files.reduce((total, file) => { + return total + Math.ceil(file.size / chunkSize); + }, 0); + + let completedChunks = 0; + if (files.length > 0) { files.forEach(file => { const fileName = file.name; const fileType = file.type const chunkSize = 1024 * 1024; // 1MB - const totalChunks = Math.ceil(file.size / chunkSize); + const amountOfChunks = Math.ceil(file.size / chunkSize); let start = 0; let end = chunkSize; - for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + for (let chunkIndex = 0; chunkIndex < amountOfChunks; chunkIndex++) { const chunk = file.slice(start, end); - uploadChunk(chunk, fileName, fileType, chunkIndex, uploadMethod); + const upload = uploadChunk(challengeId, attemptId, config, chunk, fileName, fileType, chunkIndex) + .then(() => { + completedChunks++; + const percentage = Math.round((completedChunks / totalChunks) * 100); + setUploadPercentage(percentage); + }); + uploads.push(upload); start = end; end = Math.min(start + chunkSize, file.size); @@ -103,9 +81,10 @@ function uploadFilesInChunks(formData: FormData, uploadMethod: (data: FormData) } else { console.error('No files found in FormData'); } + await Promise.all(uploads) } -async function uploadChunk(chunk: Blob, fileName: string, fileType: string, chunkIndex: number, uploadMethod: (data: FormData) => Promise>) { +async function uploadChunk(challengeId: number, attemptId: any, config: AxiosRequestConfig, chunk: Blob, fileName: string, fileType: string, chunkIndex: number) { const chunkFormData = new FormData(); chunkFormData.append('chunk', chunk); chunkFormData.append('fileName', fileName); // Send the file name for reference @@ -113,22 +92,34 @@ async function uploadChunk(chunk: Blob, fileName: string, fileType: string, chun chunkFormData.append('chunkIndex', String(chunkIndex)); try { - const response = await uploadMethod(chunkFormData) + const response = await axios.post(`/api/challenges/${challengeId}/attempt/${attemptId}`, chunkFormData, config) return response } catch (err) { - console.error(err) + console.warn('Upload failed, saving to IndexedDB for retry later'); + + + await db.uploads.add({ + challengeId, + attemptId, + chunkIndex: String(chunkIndex), + fileName, + fileType, + data: await chunk.arrayBuffer(), + }); + + // Check if sync is supported + + const registration = await navigator.serviceWorker.ready; + + if ('sync' in registration) { + registration.sync.register('sync-uploads'); + } else { + console.info('No background uploading detected') + } + + throw err; } - // uploadMethod(chunkFormData) - // .then(response => { - // if (!response.ok) { - // throw new Error('Chunk upload failed'); - // } - // console.log(`Chunk ${chunkIndex + 1} of ${totalChunks} for ${fileName} uploaded successfully.`); - // }) - // .catch(error => { - // console.error('Error uploading chunk:', error); - // }); } diff --git a/src/utils/DexieDB.ts b/src/utils/DexieDB.ts index 386290f..558e7c2 100644 --- a/src/utils/DexieDB.ts +++ b/src/utils/DexieDB.ts @@ -2,9 +2,12 @@ import Dexie from 'dexie'; interface UploadEntry { id?: number; - challengeId: string; - files: { fileName: string; fileType: string; fileData: ArrayBuffer }[]; // Store multiple files - additionalData: { [key: string]: string }; // Store other FormData fields + challengeId: number; + attemptId: string; + chunkIndex: string; + fileName: string; + fileType: string; + data: ArrayBuffer; } class UploadDatabase extends Dexie { @@ -13,7 +16,7 @@ class UploadDatabase extends Dexie { constructor() { super('UploadDB'); this.version(1).stores({ - uploads: '++id, challengeId' + uploads: '++id, challengeId, attemptId, chunkIndex', }); this.uploads = this.table('uploads'); From 683d11d4509b72d7a19dc4cb7715a166bfd4a607 Mon Sep 17 00:00:00 2001 From: kaasbroodju Date: Tue, 22 Jul 2025 16:16:13 +0200 Subject: [PATCH 4/4] Fix: type hinting service worker; as of now not all browser support the functions --- src/App.tsx | 8 ++++++-- src/services/challenge.service.tsx | 17 +++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index db8e2fd..9ab78e6 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,8 +16,12 @@ const AppRoot = () => { .catch((err) => console.log('Service Worker Registration Failed', err)); if ('SyncManager' in window) { - navigator.serviceWorker.ready.then((reg) => { - reg.sync.register('sync-uploads'); + navigator.serviceWorker.ready.then(async (registration) => { + if ('sync' in registration) { + // https://developer.mozilla.org/en-US/docs/Web/API/SyncManager + const syncManager = registration.sync as { register: (tag: string) => Promise }; + await syncManager.register('sync-uploads'); + } }); } } diff --git a/src/services/challenge.service.tsx b/src/services/challenge.service.tsx index 6a6b54f..843c350 100644 --- a/src/services/challenge.service.tsx +++ b/src/services/challenge.service.tsx @@ -1,4 +1,4 @@ -import axios, { AxiosProgressEvent, AxiosRequestConfig, AxiosResponse } from "axios"; +import axios, { AxiosRequestConfig } from "axios"; import { Challenge } from "../model/challenge.tsx"; import { db } from "../utils/DexieDB.ts"; @@ -27,7 +27,7 @@ export const uploadChallenge = async ( const chunkSize = 1024 * 1024; // 1MB - const result = await axios.post(`/api/challenges/${challenge.id}/attempt`, + const result = await axios.post(`/api/challenges/${challenge.id}/attempt`, Object.fromEntries((formData.getAll("files") as File[]).map((file: File) => [ file.name, Math.ceil(file.size / chunkSize) ])) , { headers: { Authorization: `Bearer ${localStorage.getItem('token')}` } }) @@ -42,7 +42,7 @@ export const uploadChallenge = async ( ) }; -async function uploadFilesInChunks(formData: FormData, challengeId: number, attemptId: any, config: AxiosRequestConfig, setUploadPercentage: (percentage: number) => void) { +async function uploadFilesInChunks(formData: FormData, challengeId: number, attemptId: string, config: AxiosRequestConfig, setUploadPercentage: (percentage: number) => void) { const uploads: Promise[] = [] const files = formData.getAll('files') as File[]; // Get all files from the 'files' input @@ -84,7 +84,7 @@ async function uploadFilesInChunks(formData: FormData, challengeId: number, atte await Promise.all(uploads) } -async function uploadChunk(challengeId: number, attemptId: any, config: AxiosRequestConfig, chunk: Blob, fileName: string, fileType: string, chunkIndex: number) { +async function uploadChunk(challengeId: number, attemptId: string, config: AxiosRequestConfig, chunk: Blob, fileName: string, fileType: string, chunkIndex: number) { const chunkFormData = new FormData(); chunkFormData.append('chunk', chunk); chunkFormData.append('fileName', fileName); // Send the file name for reference @@ -109,16 +109,17 @@ async function uploadChunk(challengeId: number, attemptId: any, config: AxiosReq }); // Check if sync is supported - const registration = await navigator.serviceWorker.ready; if ('sync' in registration) { - registration.sync.register('sync-uploads'); + // https://developer.mozilla.org/en-US/docs/Web/API/SyncManager + const syncManager = registration.sync as { register: (tag: string) => Promise }; + await syncManager.register('sync-uploads'); + return; } else { console.info('No background uploading detected') + throw err; } - - throw err; } }