Verify native library integrity during extraction/load (#2368)

## Description

We've hit a problem where the `CombinedRuntimeLoader` extracts native
files, but gets interrupted in the middle. This is bad, cause all
`CombinedRuntimeLoader` used to check a file was its existence. This
change uses the hash of the file to verify that it's correct. This will
be checked at runtime, everytime, if the file is extant. If this check
fails, we will delete the extant file and attempt to reextract. We also
check a newly extracted file, if that hash does not match we error.

Note that this is reliant on
PhotonVision/wpilib-tool-plugin#8 and should
follow #2367

## Meta

Merge checklist:
- [x] Pull Request title is [short, imperative
summary](https://cbea.ms/git-commit/) of proposed changes
- [x] The description documents the _what_ and _why_, including events
that led to this PR
- [ ] If this PR changes behavior or adds a feature, user documentation
is updated
- [ ] If this PR touches photon-serde, all messages have been
regenerated and hashes have not changed unexpectedly
- [ ] If this PR touches configuration, this is backwards compatible
with all settings going back to the previous seasons's last release
(seasons end after champs ends)
- [ ] If this PR touches pipeline settings or anything related to data
exchange, the frontend typing is updated
- [ ] If this PR addresses a bug, a regression test for it is added

---------

Co-authored-by: Matt M <matthew.morley.ca@gmail.com>

Author: samfreund

.github/workflows/build.yml

@@ -443,9 +443,32 @@ jobs:
       - uses: actions/download-artifact@v7
         with:
           name: ${{ matrix.artifact-name }}
-      - run: java -jar ${{ matrix.extraOpts }} *.jar --smoketest
+      # The jar is run twice to exercise different code paths.
+      - run: |
+          echo "=== First run ==="
+          java -jar ${{ matrix.extraOpts }} *.jar --smoketest
+          echo "=== Checking for files to corrupt ==="
+          find ~ -type f \( -name "*.so" -o -name "*.dylib" \) | head -20
+          if [ -d ~/.wpilib ]; then
+            echo "~/.wpilib directory exists"
+            echo "Contents of ~/.wpilib:"
+            find ~/.wpilib -type f \( -name "*.so" -o -name "*.dylib" \) | head -10
+            RANDOM_FILE=$(find ~/.wpilib -type f \( -name "*.so" -o -name "*.dylib" \) | sort -R | head -n 1)
+            if [ ! -z "$RANDOM_FILE" ]; then
+              echo "Corrupting file: $RANDOM_FILE"
+              echo "corrupted data" > "$RANDOM_FILE"
+            else
+              echo "No .so or .dylib files found in ~/.wpilib"
+            fi
+          else
+            echo "~/.wpilib directory does not exist"
+          fi
+          echo "=== Second run ==="
+          java -jar ${{ matrix.extraOpts }} *.jar --smoketest
         if: ${{ (matrix.os) != 'windows-latest' }}
-      - run: ls *.jar | %{ Write-Host "Running $($_.Name)"; Start-Process "java" -ArgumentList "-jar `"$($_.FullName)`" --smoketest" -NoNewWindow -Wait; break }
+      - run: |
+          ls *.jar | %{ Write-Host "Running $($_.Name)"; Start-Process "java" -ArgumentList "-jar `"$($_.FullName)`" --smoketest" -NoNewWindow -Wait; break }
+          ls *.jar | %{ Write-Host "Running $($_.Name)"; Start-Process "java" -ArgumentList "-jar `"$($_.FullName)`" --smoketest" -NoNewWindow -Wait; break }
         if: ${{ (matrix.os) == 'windows-latest' }}
 
   build-image:

build.gradle

@@ -5,7 +5,7 @@ plugins {
     id "com.diffplug.spotless" version "8.1.0"
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
     id "edu.wpi.first.GradleRIO" version "2026.2.1"
-    id 'org.photonvision.tools.WpilibTools' version '2.3.3-photon'
+    id 'org.photonvision.tools.WpilibTools' version '2.4.1-photon'
     id 'com.google.protobuf' version '0.9.3' apply false
     id 'edu.wpi.first.GradleJni' version '1.1.0'
     id "org.ysb33r.doxygen" version "2.0.0" apply false

photon-targeting/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -17,13 +17,13 @@
 
 package org.photonvision.common.hardware;
 
-import edu.wpi.first.util.CombinedRuntimeLoader;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.function.Supplier;
+import org.photonvision.jni.CombinedRuntimeLoader;
 
 @SuppressWarnings({"unused", "doclint"})
 public enum Platform {

photon-targeting/src/main/java/org/photonvision/jni/CombinedRuntimeLoader.java

@@ -18,24 +18,34 @@
 
 package org.photonvision.jni;
 
-import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.HexFormat;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 /** Loads dynamic libraries for all platforms. */
 public final class CombinedRuntimeLoader {
     private CombinedRuntimeLoader() {}
 
     private static String extractionDirectory;
 
+    private static final Object extractCompleteLock = new Object();
+    private static boolean extractAndVerifyComplete = false;
+    private static List<String> filesAlreadyExtracted = new CopyOnWriteArrayList<>();
+
     /**
      * Returns library extraction directory.
      *
@@ -124,6 +134,27 @@ private static String getLoadErrorMessage(String libraryName, UnsatisfiedLinkErr
         return msg.toString();
     }
 
+    /**
+     * Architecture-specific information containing file hashes for a specific CPU architecture (e.g.,
+     * x86-64, arm64).
+     */
+    public record ArchInfo(Map<String, String> fileHashes) {}
+
+    /**
+     * Platform-specific information containing architectures for a specific OS platform (e.g., linux,
+     * windows).
+     */
+    public record PlatformInfo(Map<String, ArchInfo> architectures) {}
+
+    /** Overall resource information to be serialized */
+    public record ResourceInformation(
+            // Combined MD5 hash of all native resource files
+            String hash,
+            // Platform-specific native libraries organized by platform then architecture
+            Map<String, PlatformInfo> platforms,
+            // List of supported versions for these native resources
+            List<String> versions) {}
+
     /**
      * Extract a list of native libraries.
      *
@@ -133,48 +164,65 @@ private static String getLoadErrorMessage(String libraryName, UnsatisfiedLinkErr
      * @return List of all libraries that were extracted
      * @throws IOException Thrown if resource not found or file could not be extracted
      */
-    @SuppressWarnings("unchecked")
     public static <T> List<String> extractLibraries(Class<T> clazz, String resourceName)
             throws IOException {
-        TypeReference<HashMap<String, Object>> typeRef = new TypeReference<>() {};
         ObjectMapper mapper = new ObjectMapper();
-        Map<String, Object> map;
+        ResourceInformation resourceInfo;
         try (var stream = clazz.getResourceAsStream(resourceName)) {
-            map = mapper.readValue(stream, typeRef);
+            resourceInfo = mapper.readValue(stream, ResourceInformation.class);
         }
 
         var platformPath = Paths.get(getPlatformPath());
         var platform = platformPath.getName(0).toString();
         var arch = platformPath.getName(1).toString();
 
-        var platformMap = (Map<String, List<String>>) map.get(platform);
+        var platformInfo = resourceInfo.platforms().get(platform);
+        if (platformInfo == null) {
+            throw new IOException("Platform " + platform + " not found in resource information");
+        }
+
+        var archInfo = platformInfo.architectures().get(arch);
+        if (archInfo == null) {
+            throw new IOException("Architecture " + arch + " not found for platform " + platform);
+        }
 
-        var fileList = platformMap.get(arch);
+        // Map of <file to extract> to <hash we loaded from the JSON>
+        Map<String, String> filenameToHash = archInfo.fileHashes();
 
         var extractionPathString = getExtractionDirectory();
 
         if (extractionPathString == null) {
-            String hash = (String) map.get("hash");
+            // Folder to extract to derived from overall hash
+            String combinedHash = resourceInfo.hash();
 
             var defaultExtractionRoot = getDefaultExtractionRoot();
-            var extractionPath = Paths.get(defaultExtractionRoot, platform, arch, hash);
+            var extractionPath = Paths.get(defaultExtractionRoot, platform, arch, combinedHash);
             extractionPathString = extractionPath.toString();
 
             setExtractionDirectory(extractionPathString);
         }
 
         List<String> extractedFiles = new ArrayList<>();
 
-        byte[] buffer = new byte[0x10000]; // 64K copy buffer
-
-        for (var file : fileList) {
+        for (String file : filenameToHash.keySet()) {
             try (var stream = clazz.getResourceAsStream(file)) {
                 Objects.requireNonNull(stream);
 
                 var outputFile = Paths.get(extractionPathString, new File(file).getName());
+
+                String fileHash = filenameToHash.get(file);
+
                 extractedFiles.add(outputFile.toString());
                 if (outputFile.toFile().exists()) {
-                    continue;
+                    if (hashEm(outputFile.toFile()).equals(fileHash)) {
+                        continue;
+                    } else {
+                        // Hashes don't match, delete and re-extract
+                        System.err.println(
+                                outputFile.toAbsolutePath().toString()
+                                        + " failed validation - deleting and re-extracting");
+                        outputFile.toFile().delete();
+                    }
                 }
                 var parent = outputFile.getParent();
                 if (parent == null) {
@@ -183,17 +231,32 @@ public static <T> List<String> extractLibraries(Class<T> clazz, String resourceN
                 parent.toFile().mkdirs();
 
                 try (var os = Files.newOutputStream(outputFile)) {
-                    int readBytes;
-                    while ((readBytes = stream.read(buffer)) != -1) { // NOPMD
-                        os.write(buffer, 0, readBytes);
-                    }
+                    Files.copy(stream, outputFile, StandardCopyOption.REPLACE_EXISTING);
+                }
+
+                if (!hashEm(outputFile.toFile()).equals(fileHash)) {
+                    throw new IOException("Hash of extracted file does not match expected hash");
                 }
             }
         }
 
         return extractedFiles;
     }
 
+    private static String hashEm(File f) throws IOException {
+        try {
+            MessageDigest fileHash = MessageDigest.getInstance("MD5");
+            try (var dis =
+                    new DigestInputStream(new BufferedInputStream(new FileInputStream(f)), fileHash)) {
+                dis.readAllBytes();
+            }
+            var ret = HexFormat.of().formatHex(fileHash.digest());
+            return ret;
+        } catch (NoSuchAlgorithmException e) {
+            throw new IOException("Unable to verify extracted native files", e);
+        }
+    }
+
     /**
      * Load a single library from a list of extracted files.
      *
@@ -229,12 +292,16 @@ public static void loadLibrary(String libraryName, List<String> extractedFiles)
      */
     public static <T> void loadLibraries(Class<T> clazz, String... librariesToLoad)
             throws IOException {
-        // Extract everything
-
-        var extractedFiles = extractLibraries(clazz, "/ResourceInformation.json");
+        synchronized (extractCompleteLock) {
+            if (extractAndVerifyComplete == false) {
+                // Extract everything
+                filesAlreadyExtracted = extractLibraries(clazz, "/ResourceInformation.json");
+                extractAndVerifyComplete = true;
+            }
 
-        for (var library : librariesToLoad) {
-            loadLibrary(library, extractedFiles);
+            for (var library : librariesToLoad) {
+                loadLibrary(library, filesAlreadyExtracted);
+            }
         }
     }
 }