Skip to content

[file_packager.py] Add --export-es6 to file_packager.py #24737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+102 −21
Open

[file_packager.py] Add --export-es6 to file_packager.py #24737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
be712ca
Add modularize to file_packager.py
lkwinta Jul 18, 2025
b277aea
simplify promise usage
lkwinta Aug 12, 2025
f669970
simplyfi help code
lkwinta Aug 12, 2025
ddb448e
simplyfi test
lkwinta Aug 12, 2025
529e4a2
explicite promise resolving
lkwinta Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ChangeLog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ See docs/process.md for more on how version tagging works.
- emcc will now error if `MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION` or
`MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION` are used with `SINGLE_FILE`.
These are fundamentally incompatible but were previously ignored. (#24849)
- `--export-es6` flag was added to `file_packager.py` available when run
standalone, to enable ES6 imports of generated JavaScript code (#24737)

4.0.12 - 08/01/25
-----------------
Expand Down
36 changes: 36 additions & 0 deletions test/test_other.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3919,6 +3919,11 @@ def test_file_packager_returns_error_if_target_equal_to_jsoutput(self):
err = self.expect_fail([FILE_PACKAGER, 'test.data', '--js-output=test.data'])
self.assertContained(MESSAGE, err)

def test_file_packager_returns_error_if_emcc_and_export_es6(self):
MESSAGE = 'error: Can\'t use --export-es6 option together with --from-emcc since the code should be embedded within emcc\'s code'
err = self.expect_fail([FILE_PACKAGER, 'test.data', '--export-es6', '--from-emcc'])
self.assertContained(MESSAGE, err)

def test_file_packager_embed(self):
create_file('data.txt', 'hello data')

Expand All @@ -3945,6 +3950,37 @@ def test_file_packager_embed(self):
output = self.run_js('a.out.js')
self.assertContained('hello data', output)

def test_file_packager_export_es6(self):
create_file('smth.txt', 'hello data')
self.run_process([FILE_PACKAGER, 'test.data', '--export-es6', '--preload', 'smth.txt', '--js-output=dataFileLoader.js'])

create_file('test.c', '''
#include <stdio.h>
#include <emscripten.h>

EMSCRIPTEN_KEEPALIVE int test_fun() {
FILE* f = fopen("smth.txt", "r");
char buf[64] = {0};
int rtn = fread(buf, 1, 64, f);
buf[rtn] = '\\0';
fclose(f);
printf("%s\\n", buf);
return 0;
}
''')
self.run_process([EMCC, 'test.c', '-sFORCE_FILESYSTEM', '-sMODULARIZE', '-sEXPORT_ES6', '-o', 'moduleFile.js'])

create_file('run.js', '''
import loadDataFile from './dataFileLoader.js'
import {default as loadModule} from './moduleFile.js'

var module = await loadModule();
await loadDataFile(module);
module._test_fun();
''')

self.assertContained('hello data', self.run_js('run.js'))

@crossplatform
def test_file_packager_depfile(self):
create_file('data1.txt', 'data1')
Expand Down
85 changes: 64 additions & 21 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
Usage:
file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--help]
file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--export-es6] [--help]
--preload ,
--embed See emcc --help for more details on those options.
Expand All @@ -41,6 +41,8 @@
--export-name=EXPORT_NAME Use custom export name (default is `Module`)
--export-es6 Wrap generated code inside ES6 exported function
--no-force Don't create output if no valid input file is specified.
--use-preload-cache Stores package in IndexedDB so that subsequent loads don't need to do XHR. Checks package version.
Expand Down Expand Up @@ -129,6 +131,7 @@ def __init__(self):
self.use_preload_plugins = False
self.support_node = True
self.wasm64 = False
self.export_es6 = False


class DataFile:
Expand Down Expand Up @@ -362,7 +365,7 @@ def main(): # noqa: C901, PLR0912, PLR0915
To revalidate these numbers, run `ruff check --select=C901,PLR091`.
"""
if len(sys.argv) == 1:
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--help]
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--export-es6] [--help]
Try 'file_packager --help' for more details.''')
return 1

Expand Down Expand Up @@ -391,6 +394,9 @@ def main(): # noqa: C901, PLR0912, PLR0915
elif arg == '--no-force':
options.force = False
leading = ''
elif arg == '--export-es6':
options.export_es6 = True
leading = ''
elif arg == '--use-preload-cache':
options.use_preload_cache = True
leading = ''
Expand Down Expand Up @@ -485,6 +491,11 @@ def main(): # noqa: C901, PLR0912, PLR0915
diagnostics.error('TARGET should not be the same value of --js-output')
return 1

if options.from_emcc and options.export_es6:
diagnostics.error('Can\'t use --export-es6 option together with --from-emcc since the code should be embedded '
'within emcc\'s code')
return 1

walked.append(__file__)
for file_ in data_files:
if not should_ignore(file_.srcpath):
Expand Down Expand Up @@ -621,20 +632,38 @@ def generate_js(data_target, data_files, metadata):
if options.from_emcc:
ret = ''
else:
ret = '''
if options.export_es6:
ret = 'export default async function loadDataFile(Module) {\n'
else:
ret = '''
var Module = typeof %(EXPORT_NAME)s != 'undefined' ? %(EXPORT_NAME)s : {};\n''' % {"EXPORT_NAME": options.export_name}

ret += '''
Module['expectedDataFileDownloads'] ??= 0;
Module['expectedDataFileDownloads']++;
(() => {
Module['expectedDataFileDownloads']++;'''

if not options.export_es6:
ret += '''
(() => {'''

ret += '''
// Do not attempt to redownload the virtual filesystem data when in a pthread or a Wasm Worker context.
var isPthread = typeof ENVIRONMENT_IS_PTHREAD != 'undefined' && ENVIRONMENT_IS_PTHREAD;
var isWasmWorker = typeof ENVIRONMENT_IS_WASM_WORKER != 'undefined' && ENVIRONMENT_IS_WASM_WORKER;
if (isPthread || isWasmWorker) return;\n'''

if options.support_node:
ret += " var isNode = typeof process === 'object' && typeof process.versions === 'object' && typeof process.versions.node === 'string';\n"

if options.support_node and options.export_es6:
ret += '''if (isNode) {
const { createRequire } = await import('module');
/** @suppress{duplicate} */
var require = createRequire(import.meta.url);
}\n'''

if options.export_es6:
ret += 'return new Promise((loadDataResolve, loadDataReject) => {\n'
ret += ' async function loadPackage(metadata) {\n'

code = '''
Expand Down Expand Up @@ -684,6 +713,8 @@ def generate_js(data_target, data_files, metadata):
create_data = '''// canOwn this data in the filesystem, it is a slide into the heap that will never change
Module['FS_createDataFile'](this.name, null, byteArray, true, true, true);
Module['removeRunDependency'](`fp ${that.name}`);'''
ready_promise = '''
loadDataResolve();'''

if not options.lz4:
# Data requests - for getting a block of data out of the big archive - have
Expand All @@ -710,14 +741,14 @@ def generate_js(data_target, data_files, metadata):
finish: function(byteArray) {
var that = this;
%s
this.requests[this.name] = null;
this.requests[this.name] = null;%s
}
};
var files = metadata['files'];
for (var i = 0; i < files.length; ++i) {
new DataRequest(files[i]['start'], files[i]['end'], files[i]['audio'] || 0).open('GET', files[i]['filename']);
}\n''' % (create_preloaded if options.use_preload_plugins else create_data)
}\n''' % (create_preloaded if options.use_preload_plugins else create_data, ready_promise if options.export_es6 else '')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about:

finish_handler = create_preloaded if options.use_preload_plugins else create_data
if export_es6:
   finish_handler += '\nloadDataResolve();'

Then use finish_handler here in the string template.


if options.has_embedded and not options.obj_output:
diagnostics.warn('--obj-output is recommended when using --embed. This outputs an object file for linking directly into your application is more efficient than JS encoding')
Expand Down Expand Up @@ -1020,6 +1051,10 @@ def generate_js(data_target, data_files, metadata):

code += '''
Module['preloadResults'] ??= {};\n'''
catch_case = '''
.catch((error) => {
loadDataReject(error);
})'''
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about doing:

catch_handler = ''
if export_es6:
  catch_handler = ''' ... '''

Then you don't need repeated (catch_case if options.export_es6 else '') below.


if options.use_preload_cache:
code += '''
Expand All @@ -1046,10 +1081,10 @@ def generate_js(data_target, data_files, metadata):
}
}
} catch(e) {
await preloadFallback(e);
await preloadFallback(e)%s;
}
Module['setStatus']?.('Downloading...');\n'''
Module['setStatus']?.('Downloading...');\n''' % (catch_case if options.export_es6 else '')
else:
# Not using preload cache, so we might as well start the xhr ASAP,
# potentially before JS parsing of the main codebase if it's after us.
Expand All @@ -1062,15 +1097,16 @@ def generate_js(data_target, data_files, metadata):
if (!fetched) {
// Note that we don't use await here because we want to execute the
// the rest of this function immediately.
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE).then((data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
})
}\n'''
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE)
.then((data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
})%s;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like there is no current error handling here, for this fetch. It looks like the code is trying to do a pre-fetch and maybe we can just ignore failures (like we currently do) for the pre-fetch?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In that case this block could be reverted

}\n''' % (catch_case if options.export_es6 else '')

code += '''
Module['preloadResults'][PACKAGE_NAME] = {fromCache: false};
Expand All @@ -1087,10 +1123,10 @@ def generate_js(data_target, data_files, metadata):
ret += '''
}
if (Module['calledRun']) {
runWithFS(Module);
runWithFS(Module)%s;
} else {
(Module['preRun'] ??= []).push(runWithFS); // FS is not initialized yet, wait for it
}\n'''
}\n''' % (catch_case if options.export_es6 else '')

if options.separate_metadata:
node_support_code = ''
Expand Down Expand Up @@ -1128,7 +1164,14 @@ def generate_js(data_target, data_files, metadata):
}
loadPackage(%s);\n''' % json.dumps(metadata)

ret += '''
if options.export_es6:
ret += '''
});
}
// END the loadDataFile function
'''
else:
ret += '''
})();\n'''

return ret
Expand Down