Export roman2khmer v3 model to TFLite/CoreML + exact-match shortcut bundle

Ships the frequency-oversampled (v3) model and the top-500 exact-match
shortcut layer under ml/roman2khmer/dist/ (tracked, unlike artifacts/) so
the bundle can be merged into the iOS/Android projects on another machine
without re-running training. compare_all.py showed v3+shortcut beats plain
(v2) training and the shortcut alone across every slice of the held-out set.

Also parameterizes convert_to_mobile.py's conversion routine so it can
target either model version instead of only the default path.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: RathanakSreang

ml/roman2khmer/conversion/convert_to_mobile.py

@@ -24,15 +24,15 @@
 from training.model import build_model
 
 
-def convert_tflite(model):
+def convert_tflite(model, tflite_path):
     converter = tf.lite.TFLiteConverter.from_keras_model(model)
     converter.optimizations = [tf.lite.Optimize.DEFAULT]
     tflite_bytes = converter.convert()
-    config.TFLITE_PATH.write_bytes(tflite_bytes)
+    tflite_path.write_bytes(tflite_bytes)
     return tflite_bytes
 
 
-def convert_coreml(model):
+def convert_coreml(model, coreml_path):
     @tf.function
     def f(chars, prev):
         return model({"chars": chars, "prev": prev})
@@ -51,7 +51,7 @@ def f(chars, prev):
         minimum_deployment_target=ct.target.iOS15,
         compute_precision=ct.precision.FLOAT32,
     )
-    mlmodel.save(str(config.COREML_PATH))
+    mlmodel.save(str(coreml_path))
     return mlmodel
 
 
@@ -107,8 +107,8 @@ def benchmark(fn, inputs, n=50):
     return (time.time() - t0) / n * 1000
 
 
-def main():
-    trained = keras.models.load_model(config.KERAS_MODEL_PATH, safe_mode=False)
+def convert(keras_path, tflite_path, coreml_path, dest_dir):
+    trained = keras.models.load_model(keras_path, safe_mode=False)
     vocab = load_vocab()
     context_vocab = load_context_vocab()
 
@@ -128,12 +128,12 @@ def main():
 
     keras_probs = model.predict(X_sample, verbose=0)
 
-    print("converting to TFLite...")
-    tflite_bytes = convert_tflite(model)
+    print(f"converting {keras_path.name} to TFLite...")
+    tflite_bytes = convert_tflite(model, tflite_path)
     tflite_probs = tflite_predict(tflite_bytes, X_sample)
 
-    print("converting to CoreML...")
-    mlmodel = convert_coreml(model)
+    print(f"converting {keras_path.name} to CoreML...")
+    mlmodel = convert_coreml(model, coreml_path)
 
     max_diff = np.abs(keras_probs - tflite_probs).max()
     top1 = agreement(keras_probs, tflite_probs, 1)
@@ -143,7 +143,7 @@ def main():
     tflite_latency = benchmark(lambda x: tflite_predict(tflite_bytes, x), X_sample)
     print(f"TFLite latency: {tflite_latency:.2f} ms/inference")
 
-    tflite_size_kb = config.TFLITE_PATH.stat().st_size / 1024
+    tflite_size_kb = tflite_path.stat().st_size / 1024
     print(f"TFLite size: {tflite_size_kb:.1f} KB")
 
     try:
@@ -160,15 +160,20 @@ def main():
         coreml_latency = benchmark(lambda x: coreml_predict(mlmodel, x), X_sample)
         print(f"CoreML latency: {coreml_latency:.2f} ms/inference")
 
-    shutil.copy(config.VOCAB_PATH, config.MODEL_DIR / config.VOCAB_PATH.name)
-    shutil.copy(config.CHAR_VOCAB_PATH, config.MODEL_DIR / config.CHAR_VOCAB_PATH.name)
-    shutil.copy(config.CONTEXT_VOCAB_PATH, config.MODEL_DIR / config.CONTEXT_VOCAB_PATH.name)
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    shutil.copy(config.VOCAB_PATH, dest_dir / config.VOCAB_PATH.name)
+    shutil.copy(config.CHAR_VOCAB_PATH, dest_dir / config.CHAR_VOCAB_PATH.name)
+    shutil.copy(config.CONTEXT_VOCAB_PATH, dest_dir / config.CONTEXT_VOCAB_PATH.name)
+
+    print(f"saved {tflite_path}")
+    print(f"saved {coreml_path}")
+    print(f"saved {dest_dir / config.VOCAB_PATH.name}")
+    print(f"saved {dest_dir / config.CHAR_VOCAB_PATH.name}")
+    print(f"saved {dest_dir / config.CONTEXT_VOCAB_PATH.name}")
 
-    print(f"saved {config.TFLITE_PATH}")
-    print(f"saved {config.COREML_PATH}")
-    print(f"saved {config.MODEL_DIR / config.VOCAB_PATH.name}")
-    print(f"saved {config.MODEL_DIR / config.CHAR_VOCAB_PATH.name}")
-    print(f"saved {config.MODEL_DIR / config.CONTEXT_VOCAB_PATH.name}")
+
+def main():
+    convert(config.KERAS_MODEL_PATH, config.TFLITE_PATH, config.COREML_PATH, config.MODEL_DIR)
 
 
 if __name__ == "__main__":

ml/roman2khmer/conversion/export_dist.py

@@ -0,0 +1,44 @@
+"""Produce the shippable roman2khmer bundle in dist/ (tracked in git, unlike
+artifacts/) so it can be merged into the iOS project on another machine
+without re-running training or conversion.
+
+Ships the v3 (frequency-oversampled) model, since comparison/compare_all.py
+showed it beats v2 across the board, plus the exact-match shortcut map for
+the SHORTCUT_TOP_N highest-frequency words (see comparison/shortcut.py) --
+the two combined gave the best numbers in every slice of that comparison.
+
+Usage: python -m conversion.export_dist   (run from ml/roman2khmer/)
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import config
+from conversion.convert_to_mobile import convert
+from comparison.shortcut import build_shortcut_map
+from training.dataset import load_vocab
+
+DIST_DIR = config.ROOT / "dist"
+
+
+def main():
+    DIST_DIR.mkdir(parents=True, exist_ok=True)
+
+    convert(
+        config.KERAS_MODEL_V3_PATH,
+        DIST_DIR / "Roman2Khmer.tflite",
+        DIST_DIR / "Roman2Khmer.mlpackage",
+        DIST_DIR,
+    )
+
+    vocab = load_vocab()
+    shortcut_map = build_shortcut_map(vocab, config.SHORTCUT_TOP_N)
+    with open(DIST_DIR / "shortcut.json", "w", encoding="utf-8") as f:
+        json.dump(shortcut_map, f, ensure_ascii=False, indent=2)
+    print(f"saved {DIST_DIR / 'shortcut.json'} ({len(shortcut_map)} exact romanizations)")
+
+
+if __name__ == "__main__":
+    main()

ml/roman2khmer/dist/README.md

@@ -0,0 +1,42 @@
+# roman2khmer shippable bundle
+
+Committed to git (unlike `../artifacts/`) so it can be merged into the iOS
+(and Android) projects on another machine without re-running training or
+conversion. Regenerate with `python -m conversion.export_dist` from
+`ml/roman2khmer/` if the model or vocab ever changes.
+
+This is the **v3** model (frequency-oversampled training) plus the
+exact-match shortcut for the 500 highest-frequency words -- see the
+`comparison/` scripts for how that combination was chosen over plain (v2)
+training and over the shortcut alone.
+
+## Files
+
+- `Roman2Khmer.tflite` / `Roman2Khmer.mlpackage` -- the model, two inputs:
+  - `chars`: int32[20], the romanized text typed so far, encoded via
+    `char_vocab.json` (`pad_index`/`unk_index`/`chars` map, 0-padded on the right)
+  - `prev`: int32 scalar, the previous word, encoded via `context_vocab.json`
+    (index 0 = `<unk>` -- no reliable previous-word context, e.g. first word
+    of a message; index 1 = `<s>` -- explicit sentence start)
+  - output: float32[6782] softmax over `vocab.json` (index -> Khmer word)
+- `vocab.json`, `char_vocab.json`, `context_vocab.json` -- the encode/decode
+  tables the two inputs/output above are defined against.
+- `shortcut.json` -- `{romanization: khmer_word}` for the 500 highest-
+  frequency words' exact known spellings. Apply this **before** the model,
+  and **only** once the user has finished typing a word (not mid-prefix --
+  a partial prefix can coincide character-for-character with one of these
+  words' full spelling and wrongly hijack the prediction otherwise, see
+  `comparison/shortcut.py`'s docstring for the measured collision rate):
+  if the typed text is an exact key in this map, use its value as the
+  top-1 suggestion and let the model fill in the rest of the ranked list.
+
+## Known limitations (not yet addressed by this bundle)
+
+- Offline top-1/top-3/top-5 accuracy on held-out data is roughly 31%/45%/50%
+  overall (see `comparison/compare_all.py` output) -- useful as a ranked
+  suggestion-bar candidate, not a silent auto-correct/auto-commit.
+- CoreML inference was exported but not runtime-validated (coremltools can
+  only run predictions on macOS); validate on a Mac/iOS device before
+  shipping.
+- Not yet wired into `KhmerlangCorrector`/`SuggestionProvider` (iOS) or the
+  Android equivalent -- that integration is a separate follow-up.

ml/roman2khmer/dist/Roman2Khmer.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "38607878-7a33-499e-8da5-cb01dd8ccfec": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        },
+        "e5dfb06c-5251-41c7-8b5f-0f4221dc225d": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        }
+    },
+    "rootModelIdentifier": "e5dfb06c-5251-41c7-8b5f-0f4221dc225d"
+}

ml/roman2khmer/dist/Roman2Khmer.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,33 @@
+{
+  "pad_index": 0,
+  "unk_index": 1,
+  "chars": {
+    "a": 2,
+    "b": 3,
+    "c": 4,
+    "d": 5,
+    "e": 6,
+    "f": 7,
+    "g": 8,
+    "h": 9,
+    "i": 10,
+    "j": 11,
+    "k": 12,
+    "l": 13,
+    "m": 14,
+    "n": 15,
+    "o": 16,
+    "p": 17,
+    "q": 18,
+    "r": 19,
+    "s": 20,
+    "t": 21,
+    "u": 22,
+    "v": 23,
+    "w": 24,
+    "x": 25,
+    "y": 26,
+    "z": 27
+  },
+  "max_len": 20
+}