From 78586c3c6397c823bacabb49bb562df2819e81aa Mon Sep 17 00:00:00 2001 From: Umar Getagazov Date: Wed, 30 Jul 2025 18:50:44 +0100 Subject: [PATCH 1/3] Fix IDBFS not auto-persisting on directory creation FS.mkdir does not use mkdir under the hood but rather mknod. The patched version of mknod installed by IDBFS only invokes IDBFS.queuePersist() when the node's stream is closed, which never happens for mkdir as the handle is immediately discarded and no stream is ever created. Furthermore, the bare fact of file/directory creation should be persisted even if no writes are made to the node. The test doesn't catch this because reconcilation happens in a JavaScript task, and no tasks or microtasks can execute while code is executing. IDBFS.syncfs only has a chance to run after the entirety of the test function finishes executing, at which point all of file/directory operations have been executed and persisted in memory. As long as one of those operations calls IDBFS.queuePersist, it ensures that the whole memory filesystem will get persisted to IDB and therefore shadows missing calls to IDBFS.queuePersist in other operations. Remove the mkdir node op patching (which has never been a real node op) and make mknod queue a persist operation as soon as the node is created without waiting for the caller to interact with a stream. --- src/lib/libidbfs.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/libidbfs.js b/src/lib/libidbfs.js index ff3361d5830bb..7a20f59e0fa56 100644 --- a/src/lib/libidbfs.js +++ b/src/lib/libidbfs.js @@ -85,10 +85,12 @@ addToLibrary({ if (n.memfs_stream_ops.close) return n.memfs_stream_ops.close(stream); }; + // Persist the node we just created to IndexedDB + IDBFS.queuePersist(mnt.mount); + return node; }; // Also kick off persisting the filesystem on other operations that modify the filesystem. - mnt.node_ops.mkdir = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.mkdir(...args)); mnt.node_ops.rmdir = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.rmdir(...args)); mnt.node_ops.symlink = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.symlink(...args)); mnt.node_ops.unlink = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.unlink(...args)); From f9ce9e73e19a18287b2f1b98e8617ded9bb83d22 Mon Sep 17 00:00:00 2001 From: Umar Getagazov Date: Tue, 5 Aug 2025 02:38:53 +0100 Subject: [PATCH 2/3] Fix idbPersistState being undefined on the actual mount --- src/lib/libidbfs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/libidbfs.js b/src/lib/libidbfs.js index 7a20f59e0fa56..417b9c757b401 100644 --- a/src/lib/libidbfs.js +++ b/src/lib/libidbfs.js @@ -54,7 +54,7 @@ addToLibrary({ // If the automatic IDBFS persistence option has been selected, then automatically persist // all modifications to the filesystem as they occur. if (mount?.opts?.autoPersist) { - mnt.idbPersistState = 0; // IndexedDB sync starts in idle state + mount.idbPersistState = 0; // IndexedDB sync starts in idle state var memfs_node_ops = mnt.node_ops; mnt.node_ops = {...mnt.node_ops}; // Clone node_ops to inject write tracking mnt.node_ops.mknod = (parent, name, mode, dev) => { From ab29008c04add774d54e98f91ca8e68fe3943bd6 Mon Sep 17 00:00:00 2001 From: Umar Getagazov Date: Tue, 5 Aug 2025 02:40:48 +0100 Subject: [PATCH 3/3] Add a test for IDBFS auto persistence --- test/fs/test_idbfs_autopersist.c | 245 +++++++++++++++++++++++++++++++ test/test_browser.py | 13 ++ 2 files changed, 258 insertions(+) create mode 100644 test/fs/test_idbfs_autopersist.c diff --git a/test/fs/test_idbfs_autopersist.c b/test/fs/test_idbfs_autopersist.c new file mode 100644 index 0000000000000..997f9d3fc7614 --- /dev/null +++ b/test/fs/test_idbfs_autopersist.c @@ -0,0 +1,245 @@ +/* + * Copyright 2025 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +EM_JS_DEPS(deps, "$callUserCallback"); + +enum { + TEST_CASE_OPEN, + TEST_CASE_CLOSE, + TEST_CASE_SYMLINK, + TEST_CASE_UNLINK, + TEST_CASE_RENAME, + TEST_CASE_MKDIR, +}; + +static void test_case_open(void) { + switch (TEST_PHASE) { + case 1: { + int fd = open("/working1/file", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd != -1); + break; + } + case 2: { + struct stat st; + int res = lstat("/working1/file", &st); + assert(res == 0); + assert(st.st_size == 0); + break; + } + default: + assert(false); + } +} + +static void test_case_close(void) { + switch (TEST_PHASE) { + case 1: { + int fd = open("/working1/file", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd != -1); + break; + } + case 2: { + int fd = open("/working1/file", O_RDWR | O_CREAT, 0777); + assert(fd != -1); + ssize_t bytes_written = write(fd, "foo", 3); + assert(bytes_written == 3); + int res = close(fd); + assert(res == 0); + break; + } + case 3: { + struct stat st; + int res = lstat("/working1/file", &st); + assert(res == 0); + assert(st.st_size == 3); + break; + } + default: + assert(false); + } +} + +static void test_case_symlink(void) { + switch (TEST_PHASE) { + case 1: { + int fd = open("/working1/file", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd != -1); + break; + } + case 2: { + int res = symlink("/working1/file", "/working1/symlink"); + assert(res == 0); + break; + } + case 3: { + struct stat st; + int res = lstat("/working1/symlink", &st); + assert(res == 0); + break; + } + default: + assert(false); + } +} + +static void test_case_unlink(void) { + switch (TEST_PHASE) { + case 1: { + int fd = open("/working1/file", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd != -1); + break; + } + case 2: { + int res = unlink("/working1/file"); + assert(res == 0); + break; + } + case 3: { + struct stat st; + int res = lstat("/working1/file", &st); + assert(res == -1); + assert(errno == ENOENT); + break; + } + default: + assert(false); + } +} + +static void test_case_rename(void) { + switch (TEST_PHASE) { + case 1: { + int fd = open("/working1/file", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd != -1); + break; + } + case 2: { + int res = rename("/working1/file", "/working1/file_renamed"); + assert(res == 0); + break; + } + case 3: { + struct stat st; + int res = lstat("/working1/file_renamed", &st); + assert(res == 0); + res = lstat("/working1/file", &st); + assert(res == -1); + assert(errno == ENOENT); + break; + } + default: + assert(false); + } +} + +static void test_case_mkdir(void) { + switch (TEST_PHASE) { + case 1: { + int res = mkdir("/working1/dir", 0777); + assert(res == 0); + break; + } + case 2: { + struct stat st; + int res = lstat("/working1/dir", &st); + assert(res == 0); + break; + } + default: + assert(false); + } +} + +EMSCRIPTEN_KEEPALIVE +void finish(void) { + emscripten_force_exit(0); +} + +EMSCRIPTEN_KEEPALIVE +void test(void) { + switch (TEST_CASE) { + case TEST_CASE_OPEN: test_case_open(); break; + case TEST_CASE_CLOSE: test_case_close(); break; + case TEST_CASE_SYMLINK: test_case_symlink(); break; + case TEST_CASE_UNLINK: test_case_unlink(); break; + case TEST_CASE_RENAME: test_case_rename(); break; + case TEST_CASE_MKDIR: test_case_mkdir(); break; + default: assert(false); + } + + EM_ASM({ + // Wait until IDBFS has persisted before exiting + runOnceIDBFSIdle(() => { + callUserCallback(_finish); + }); + }); +} + +int main(void) { + EM_ASM({ + globalThis.runOnceIDBFSIdle = (callback) => { + const { mount } = FS.lookupPath('/working1').node; + assert('idbPersistState' in mount, 'mount object must have idbPersistState'); + if (mount.idbPersistState !== 0) { + // IDBFS hasn't finished persisting. Check again after all pending tasks have executed + setTimeout(() => runOnceIDBFSIdle(callback), 0); + return; + } + callback(); + }; + + FS.mkdir('/working1'); + FS.mount(IDBFS, { + autoPersist: true + }, '/working1'); + }); + + if (TEST_PHASE == 1) { + EM_ASM({ + // The first phase of a test case must start from an empty filesystem. + // Erase persisted state by overwriting the contents of IndexedDB + // with our empty in-memory filesystem. + FS.syncfs(false, (err) => { + assert(!err); + callUserCallback(_test); + }); + }); + } else if (TEST_PHASE > 1) { + EM_ASM({ + // All subsequent phases rely on the effects of phases before them. + // Load the persisted filesystem from IndexedDB into memory. + FS.syncfs(true, (err) => { + assert(!err); + + // FS.syncfs() may run operations on the in-memory filesystem which + // might trigger IDBFS.queuePersist() calls. These queued calls will + // also persist modifications made by the test. We want to verify that + // each operation we test calls IDBFS.queuePersist() on its own, so + // the interference from FS.syncfs() is unwanted. + // Wait until the IDBFS mount has been persisted. + runOnceIDBFSIdle(() => { + callUserCallback(_test); + }); + }); + }); + } else { + assert(false); + } + + emscripten_exit_with_live_runtime(); + return 0; +} diff --git a/test/test_browser.py b/test/test_browser.py index ef1d7ad8d54bb..d0ee02859b98a 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -1353,6 +1353,19 @@ def test_fs_idbfs_sync(self, args): print('done first half') self.btest_exit('fs/test_idbfs_sync.c', cflags=['-lidbfs.js', f'-DSECRET="{secret}"', '-sEXPORTED_FUNCTIONS=_main,_test,_finish', '-lidbfs.js'] + args) + @parameterized({ + 'open': ('TEST_CASE_OPEN', 2), + 'close': ('TEST_CASE_CLOSE', 3), + 'symlink': ('TEST_CASE_SYMLINK', 3), + 'unlink': ('TEST_CASE_UNLINK', 3), + 'rename': ('TEST_CASE_RENAME', 3), + 'mkdir': ('TEST_CASE_MKDIR', 2), + }) + def test_fs_idbfs_autopersist(self, test_case, phase_count): + self.cflags += ['-lidbfs.js', f'-DTEST_CASE={test_case}'] + for phase in range(phase_count): + self.btest_exit('fs/test_idbfs_autopersist.c', cflags=[f'-DTEST_PHASE={phase + 1}']) + def test_fs_idbfs_fsync(self): # sync from persisted state into memory before main() self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall')