Tweak flash interface (#7)

This also fixes a mistake where we padded with 0 not 0xff.

On the implementation but not the types I've left open the possibility of passing a MemoryMap so I can experiment with it as an option.

MemoryMap imports remain a bit of a disaster but one to investigate further another day.

For the Python Editor this will mean that we always return a hex from the file system and then parse it for the partial flashing case. But that seems a much more reasonable API and the conversion should be quick enough. If we care for perf then we can work towards using the MemoryMap built by microbit-fs directly. There's no reason to be going via padded bytes as that drops the UICR data which it would be better to flash.

Author: microbit-matt-hillsdon

lib/board-id.ts

@@ -61,4 +61,13 @@ export class BoardId {
   static parse(value: string): BoardId {
     return new BoardId(parseInt(value, 16));
   }
+
+  static forVersion(boardVersion: BoardVersion): BoardId {
+    switch (boardVersion) {
+      case "V1":
+        return this.v1Normalized;
+      case "V2":
+        return this.v2Normalized;
+    }
+  }
 }

lib/device.ts

@@ -4,7 +4,6 @@
  * SPDX-License-Identifier: MIT
  */
 import { TypedEventTarget } from "./events.js";
-import { BoardId } from "./board-id.js";
 
 /**
  * Specific identified error types.
@@ -94,26 +93,9 @@ export enum ConnectionStatus {
 
 export class FlashDataError extends Error {}
 
-export interface FlashDataSource {
-  /**
-   * For now we only support partially flashing contiguous data.
-   * This can be generated from microbit-fs directly (via getIntelHexBytes())
-   * or from an existing Intel Hex via slicePad.
-   *
-   * This interface is quite confusing and worth revisiting.
-   *
-   * @param boardId the id of the board.
-   * @throws FlashDataError if we cannot generate hex data.
-   */
-  partialFlashData(boardId: BoardId): Promise<Uint8Array>;
-
-  /**
-   * @param boardId the id of the board.
-   * @returns A board-specific (non-universal) Intel Hex file for the given board id.
-   * @throws FlashDataError if we cannot generate hex data.
-   */
-  fullFlashData(boardId: BoardId): Promise<string>;
-}
+export type FlashDataSource = (
+  boardVersion: BoardVersion,
+) => Promise<string | Uint8Array>;
 
 export interface ConnectOptions {
   serial?: boolean;

lib/hex-flash-data-source.ts

@@ -1,59 +1,33 @@
-import MemoryMap from "nrf-intel-hex";
 import { BoardId } from "./board-id.js";
-import { FlashDataSource, FlashDataError as FlashDataError } from "./device.js";
+import {
+  FlashDataError as FlashDataError,
+  BoardVersion,
+  FlashDataSource,
+} from "./device.js";
 import {
   isUniversalHex,
   separateUniversalHex,
 } from "@microbit/microbit-universal-hex";
 
-export class HexFlashDataSource implements FlashDataSource {
-  constructor(private hex: string) {}
-
-  partialFlashData(boardId: BoardId): Promise<Uint8Array> {
-    // Perhaps this would make more sense if we returned a MemoryMap?
-    // Then the partial flashing code could be given everything including UICR without
-    // passing a very large Uint8Array.
-
-    // Or use MM inside PF and return a (partial) hex string in the microbit-fs case?
-
-    const part = this.matchingPart(boardId);
-
-    // Cludge for a packaging issue
-    const fromHex: (
-      hexText: string,
-      maxBlockSize?: number,
-    ) => MemoryMap.default =
-      (MemoryMap as any).fromHex ?? MemoryMap.default.fromHex;
-
-    const hex = fromHex(part);
-    const keys = Array.from(hex.keys()).filter((k) => k < 0x10000000);
-    const lastKey = keys[keys.length - 1];
-    if (lastKey === undefined) {
-      throw new FlashDataError("Empty hex");
-    }
-    const lastPart = hex.get(lastKey);
-    if (!lastPart) {
-      throw new FlashDataError("Empty hex");
-    }
-    const length = lastKey + lastPart.length;
-    const data = hex.slicePad(0, length, 0);
-    return Promise.resolve(data);
-  }
-
-  fullFlashData(boardId: BoardId): Promise<string> {
-    const part = this.matchingPart(boardId);
-    return Promise.resolve(part);
-  }
-
-  private matchingPart(boardId: BoardId): string {
-    if (isUniversalHex(this.hex)) {
-      const parts = separateUniversalHex(this.hex);
-      const matching = parts.find((p) => p.boardId == boardId.normalize().id);
+/**
+ * A flash data source that converts universal hex files as needed.
+ *
+ * @param universalHex A hex file, potentially universal.
+ */
+export const createUniversalHexFlashDataSource = (
+  universalHex: string,
+): FlashDataSource => {
+  return (boardVersion: BoardVersion) => {
+    if (isUniversalHex(universalHex)) {
+      const parts = separateUniversalHex(universalHex);
+      const matching = parts.find(
+        (p) => p.boardId == BoardId.forVersion(boardVersion).id,
+      );
       if (!matching) {
         throw new FlashDataError("No matching part");
       }
-      return matching.hex;
+      return Promise.resolve(matching.hex);
     }
-    return this.hex;
-  }
-}
+    return Promise.resolve(universalHex);
+  };
+};

lib/index.ts

@@ -19,13 +19,13 @@ import {
   SerialErrorEvent,
   SerialResetEvent,
 } from "./device.js";
-import { HexFlashDataSource } from "./hex-flash-data-source.js";
+import { createUniversalHexFlashDataSource } from "./hex-flash-data-source.js";
 
 export {
   MicrobitWebUSBConnection,
   MicrobitWebBluetoothConnection,
   BoardId,
-  HexFlashDataSource,
+  createUniversalHexFlashDataSource,
   AfterRequestDevice,
   BeforeRequestDevice,
   ConnectionStatus,

lib/usb-partial-flashing.ts

@@ -47,16 +47,16 @@
 import { DAPLink } from "dapjs";
 import { Logging } from "./logging.js";
 import { withTimeout, TimeoutError } from "./async-util.js";
-import { BoardId } from "./board-id.js";
 import { DAPWrapper } from "./usb-device-wrapper.js";
-import { FlashDataSource } from "./device.js";
 import {
   CoreRegister,
   onlyChanged,
   Page,
   pageAlignBlocks,
   read32FromUInt8Array,
 } from "./usb-partial-flashing-utils.js";
+import MemoryMap from "nrf-intel-hex";
+import { BoardVersion } from "./device.js";
 
 type ProgressCallback = (n: number, partial: boolean) => void;
 
@@ -102,6 +102,7 @@ export class PartialFlashing {
   constructor(
     private dapwrapper: DAPWrapper,
     private logging: Logging,
+    private boardVersion: BoardVersion,
   ) {}
 
   private log(v: any): void {
@@ -204,11 +205,11 @@ export class PartialFlashing {
   // Falls back to a full flash if partial flashing fails.
   // Drawn from https://github.com/microsoft/pxt-microbit/blob/dec5b8ce72d5c2b4b0b20aafefce7474a6f0c7b2/editor/extension.tsx#L335
   private async partialFlashAsync(
-    boardId: BoardId,
-    dataSource: FlashDataSource,
+    data: string | Uint8Array | MemoryMap.default,
     updateProgress: ProgressCallback,
   ): Promise<boolean> {
-    const flashBytes = await dataSource.partialFlashData(boardId);
+    const flashBytes = this.convertDataToPaddedBytes(data);
+
     const checksums = await this.getFlashChecksumsAsync();
     await this.dapwrapper.writeBlockAsync(loadAddr, flashPageBIN);
     let aligned = pageAlignBlocks(flashBytes, 0, this.dapwrapper.pageSize);
@@ -219,7 +220,7 @@ export class PartialFlashing {
     let partial: boolean | undefined;
     if (aligned.length > totalPages / 2) {
       try {
-        await this.fullFlashAsync(boardId, dataSource, updateProgress);
+        await this.fullFlashAsync(data, updateProgress);
         partial = false;
       } catch (e) {
         this.log(e);
@@ -234,7 +235,7 @@ export class PartialFlashing {
       } catch (e) {
         this.log(e);
         this.log("Partial flash failed, attempting full flash.");
-        await this.fullFlashAsync(boardId, dataSource, updateProgress);
+        await this.fullFlashAsync(data, updateProgress);
         partial = false;
       }
     }
@@ -250,8 +251,7 @@ export class PartialFlashing {
 
   // Perform full flash of micro:bit's ROM using daplink.
   async fullFlashAsync(
-    boardId: BoardId,
-    dataSource: FlashDataSource,
+    data: string | Uint8Array | MemoryMap.default,
     updateProgress: ProgressCallback,
   ) {
     this.log("Full flash");
@@ -261,7 +261,7 @@ export class PartialFlashing {
     };
     this.dapwrapper.daplink.on(DAPLink.EVENT_PROGRESS, fullFlashProgress);
     try {
-      const data = await dataSource.fullFlashData(boardId);
+      data = this.convertDataToHexString(data);
       await this.dapwrapper.transport.open();
       await this.dapwrapper.daplink.flash(new TextEncoder().encode(data));
       this.logging.event({
@@ -279,8 +279,7 @@ export class PartialFlashing {
   // Flash the micro:bit's ROM with the provided image, resetting the micro:bit first.
   // Drawn from https://github.com/microsoft/pxt-microbit/blob/dec5b8ce72d5c2b4b0b20aafefce7474a6f0c7b2/editor/extension.tsx#L439
   async flashAsync(
-    boardId: BoardId,
-    dataSource: FlashDataSource,
+    data: string | Uint8Array | MemoryMap.default,
     updateProgress: ProgressCallback,
   ): Promise<boolean> {
     let resetPromise = (async () => {
@@ -300,11 +299,7 @@ export class PartialFlashing {
         await withTimeout(resetPromise, 1000);
 
         this.log("Begin flashing");
-        return await this.partialFlashAsync(
-          boardId,
-          dataSource,
-          updateProgress,
-        );
+        return await this.partialFlashAsync(data, updateProgress);
       } catch (e) {
         if (e instanceof TimeoutError) {
           this.log("Resetting micro:bit timed out");
@@ -313,7 +308,7 @@ export class PartialFlashing {
             type: "WebUSB-info",
             message: "flash-failed/attempting-full-flash",
           });
-          await this.fullFlashAsync(boardId, dataSource, updateProgress);
+          await this.fullFlashAsync(data, updateProgress);
           return false;
         } else {
           throw e;
@@ -324,4 +319,56 @@ export class PartialFlashing {
       await this.dapwrapper.disconnectAsync();
     }
   }
+
+  private convertDataToHexString(
+    data: string | Uint8Array | MemoryMap.default,
+  ): string {
+    if (typeof data === "string") {
+      return data;
+    }
+    if (data instanceof Uint8Array) {
+      return this.paddedBytesToHexString(data);
+    }
+    return data.asHexString();
+  }
+
+  private convertDataToPaddedBytes(
+    data: string | Uint8Array | MemoryMap.default,
+  ): Uint8Array {
+    if (data instanceof Uint8Array) {
+      return data;
+    }
+    if (typeof data === "string") {
+      return this.hexStringToPaddedBytes(data);
+    }
+    return this.memoryMapToPaddedBytes(data);
+  }
+
+  private hexStringToPaddedBytes(hex: string): Uint8Array {
+    // Cludge for a packaging issue
+    const fromHex: (
+      hexText: string,
+      maxBlockSize?: number,
+    ) => MemoryMap.default =
+      (MemoryMap as any).fromHex ?? MemoryMap.default.fromHex;
+
+    return this.memoryMapToPaddedBytes(fromHex(hex));
+  }
+
+  private paddedBytesToHexString(data: Uint8Array): string {
+    // Cludge for a packaging issue
+    const fromPaddedUint8Array: (data: Uint8Array) => MemoryMap.default =
+      (MemoryMap as any).fromPaddedUint8Array ??
+      MemoryMap.default.fromPaddedUint8Array;
+
+    return fromPaddedUint8Array(data).asHexString();
+  }
+
+  private memoryMapToPaddedBytes(memoryMap: MemoryMap.default): Uint8Array {
+    const flashSize = {
+      V1: 256 * 1024,
+      V2: 512 * 1024,
+    }[this.boardVersion];
+    return memoryMap.slicePad(0, flashSize);
+  }
 }

lib/usb.ts

@@ -231,13 +231,19 @@ export class MicrobitWebUSBConnection
     const progress = options.progress || (() => {});
 
     const boardId = this.connection.boardSerialInfo.id;
-    const flashing = new PartialFlashing(this.connection, this.logging);
+    const boardVersion = boardId.toBoardVersion();
+    const data = await dataSource(boardVersion);
+    const flashing = new PartialFlashing(
+      this.connection,
+      this.logging,
+      boardVersion,
+    );
     let wasPartial: boolean = false;
     try {
       if (partial) {
-        wasPartial = await flashing.flashAsync(boardId, dataSource, progress);
+        wasPartial = await flashing.flashAsync(data, progress);
       } else {
-        await flashing.fullFlashAsync(boardId, dataSource, progress);
+        await flashing.fullFlashAsync(data, progress);
       }
     } finally {
       progress(undefined, wasPartial);

src/demo.ts

@@ -5,7 +5,7 @@
  */
 import "./demo.css";
 import { MicrobitWebUSBConnection } from "../lib/usb";
-import { HexFlashDataSource } from "../lib/hex-flash-data-source";
+import { createUniversalHexFlashDataSource } from "../lib/hex-flash-data-source";
 import {
   BackgroundErrorEvent,
   ConnectionStatus,
@@ -128,7 +128,7 @@ flash.addEventListener("click", async () => {
   if (file) {
     const text = await file.text();
     if (connection.flash) {
-      await connection.flash(new HexFlashDataSource(text), {
+      await connection.flash(createUniversalHexFlashDataSource(text), {
         partial: true,
         progress: (percentage: number | undefined) => {
           console.log(percentage);