diff --git a/Cargo.lock b/Cargo.lock index 9b88747..731716e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,6 +359,26 @@ dependencies = [ "half", ] +[[package]] +name = "clang" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c044c781163c001b913cd018fc95a628c50d0d2dfea8bca77dad71edb16e37" +dependencies = [ + "clang-sys", + "libc", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", +] + [[package]] name = "clap" version = "4.6.1" @@ -587,6 +607,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ed25519" version = "2.2.3" @@ -787,6 +813,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "glyphnet-cli" version = "0.1.0" @@ -795,6 +827,7 @@ dependencies = [ "clap", "ed25519-dalek", "glyphnet-core", + "glyphnet-cv", "glyphnet-decode", "glyphnet-encode", "glyphnet-render", @@ -900,6 +933,7 @@ dependencies = [ "glyphnet-testkit", "image", "js-sys", + "opencv", "rand 0.8.6", "rayon", "thiserror 2.0.18", @@ -1373,6 +1407,39 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opencv" +version = "0.98.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c607a407be5ff2484f55d2eb289bffd01de84f962779b8470e76f035dd3563d" +dependencies = [ + "cc", + "dunce", + "jobserver", + "libc", + "num-traits", + "opencv-binding-generator", + "pkg-config", + "semver", + "shlex", + "vcpkg", + "windows", +] + +[[package]] +name = "opencv-binding-generator" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833f00c6deee8dd615249af42fa35ff030c5c73ee3c13e44baf1135a4d57af86" +dependencies = [ + "clang", + "clang-sys", + "dunce", + "percent-encoding", + "regex", + "shlex", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1410,6 +1477,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1426,6 +1499,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plotters" version = "0.3.7" @@ -2162,6 +2241,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2331,12 +2416,107 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -2370,6 +2550,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/apps/expo-glyphnet/src/features/encode/EncodePanel.tsx b/apps/expo-glyphnet/src/features/encode/EncodePanel.tsx index 52f6936..a921246 100644 --- a/apps/expo-glyphnet/src/features/encode/EncodePanel.tsx +++ b/apps/expo-glyphnet/src/features/encode/EncodePanel.tsx @@ -1,4 +1,4 @@ -import * as FileSystem from "expo-file-system"; +import { Directory, File, Paths } from "expo-file-system"; import * as MediaLibrary from "expo-media-library"; import * as Print from "expo-print"; import { useMemo, useState } from "react"; @@ -7,24 +7,6 @@ import { SvgXml } from "react-native-svg"; import { scannerAdapter } from "@/adapters/scanner"; import { Pressable, Text, TextInput, View } from "@/tw"; -function getWritableBaseDir(): string | null { - const fsAny = FileSystem as unknown as { - documentDirectory?: string | null; - cacheDirectory?: string | null; - Paths?: { - document?: { uri?: string }; - cache?: { uri?: string }; - }; - }; - return ( - fsAny.documentDirectory ?? - fsAny.cacheDirectory ?? - fsAny.Paths?.document?.uri ?? - fsAny.Paths?.cache?.uri ?? - null - ); -} - export function EncodePanel() { const [payload, setPayload] = useState("hello glyphnet"); const [svgPreview, setSvgPreview] = useState(""); @@ -63,15 +45,12 @@ export function EncodePanel() { return; } try { - const dir = getWritableBaseDir(); - if (!dir) { - setActionMessage("Unable to access local storage."); - return; - } - const uri = `${dir}glyphnet-${Date.now()}.svg`; - await FileSystem.writeAsStringAsync(uri, svgPreview, { - encoding: FileSystem.EncodingType.UTF8, - }); + const exportDir = new Directory(Paths.document, "glyphnet-exports"); + exportDir.create({ idempotent: true, intermediates: true }); + const file = new File(exportDir, `glyphnet-${Date.now()}.svg`); + file.create({ overwrite: true, intermediates: true }); + file.write(svgPreview, { encoding: "utf8" }); + const uri = file.uri; let savedToLibrary = false; try { const perm = await MediaLibrary.requestPermissionsAsync(); diff --git a/apps/expo-glyphnet/src/features/scan/ScanPanel.tsx b/apps/expo-glyphnet/src/features/scan/ScanPanel.tsx index 2d18587..62e5765 100644 --- a/apps/expo-glyphnet/src/features/scan/ScanPanel.tsx +++ b/apps/expo-glyphnet/src/features/scan/ScanPanel.tsx @@ -1,6 +1,6 @@ import { CameraView, useCameraPermissions } from "expo-camera"; +import { Directory, File, Paths } from "expo-file-system"; import * as NavigationBar from "expo-navigation-bar"; -import * as FileSystem from "expo-file-system"; import * as ImageManipulator from "expo-image-manipulator"; import { useMemo, useRef, useState } from "react"; import { Modal, Vibration } from "react-native"; @@ -16,24 +16,6 @@ const MODES = ["print", "screen", "burst"] as const; // Wide ribbon-like scan guide (GlyphNet print profile), centered in portrait view. const GUIDE_ROI = { x: 0.06, y: 0.34, w: 0.88, h: 0.26 } as const; -function getWritableBaseDir(): string | null { - const fsAny = FileSystem as unknown as { - documentDirectory?: string | null; - cacheDirectory?: string | null; - Paths?: { - document?: { uri?: string }; - cache?: { uri?: string }; - }; - }; - return ( - fsAny.documentDirectory ?? - fsAny.cacheDirectory ?? - fsAny.Paths?.document?.uri ?? - fsAny.Paths?.cache?.uri ?? - null - ); -} - export function ScanPanel() { const cameraRef = useRef(null); const insets = useSafeAreaInsets(); @@ -161,22 +143,13 @@ export function ScanPanel() { return; } try { - const dir = getWritableBaseDir(); - if (!dir) { - setDebugSaveMessage("No writable app directory available."); - return; - } const stamp = Date.now(); - const outDir = `${dir}glyphnet-debug-${stamp}/`; - await FileSystem.makeDirectoryAsync(outDir, { - intermediates: true, - }); + const outDir = new Directory(Paths.document, `glyphnet-debug-${stamp}`); + outDir.create({ idempotent: true, intermediates: true }); - const captureOut = `${outDir}capture.jpg`; - await FileSystem.copyAsync({ - from: lastCaptureUri, - to: captureOut, - }); + const captureOut = new File(outDir, "capture.jpg"); + captureOut.create({ overwrite: true, intermediates: true }); + new File(lastCaptureUri).copy(captureOut, { overwrite: true }); const crop = { originX: Math.max( @@ -198,8 +171,9 @@ export function ScanPanel() { [{ crop }], { compress: 1, format: ImageManipulator.SaveFormat.JPEG }, ); - const roiOut = `${outDir}roi.jpg`; - await FileSystem.copyAsync({ from: cropped.uri, to: roiOut }); + const roiOut = new File(outDir, "roi.jpg"); + roiOut.create({ overwrite: true, intermediates: true }); + new File(cropped.uri).copy(roiOut, { overwrite: true }); const debugJson = { ts: stamp, @@ -213,19 +187,16 @@ export function ScanPanel() { }, result: parsedResult ?? result, }; - const jsonOut = `${outDir}result.json`; - await FileSystem.writeAsStringAsync( - jsonOut, - JSON.stringify(debugJson, null, 2), - { - encoding: FileSystem.EncodingType.UTF8, - }, - ); + const jsonOut = new File(outDir, "result.json"); + jsonOut.create({ overwrite: true, intermediates: true }); + jsonOut.write(JSON.stringify(debugJson, null, 2), { + encoding: "utf8", + }); try { const Sharing = await import("expo-sharing"); if (await Sharing.isAvailableAsync()) { - await Sharing.shareAsync(roiOut, { + await Sharing.shareAsync(roiOut.uri, { mimeType: "image/jpeg", dialogTitle: "Share GlyphNet ROI debug image", }); @@ -234,7 +205,7 @@ export function ScanPanel() { // no-op: sharing may be unavailable in current runtime. } - setDebugSaveMessage(`Saved debug bundle: ${outDir}`); + setDebugSaveMessage(`Saved debug bundle: ${outDir.uri}`); } catch (error) { setDebugSaveMessage( error instanceof Error diff --git a/crates/glyphnet-cli/Cargo.toml b/crates/glyphnet-cli/Cargo.toml index 0ef7533..68c20c0 100644 --- a/crates/glyphnet-cli/Cargo.toml +++ b/crates/glyphnet-cli/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true glyphnet-core = { path = "../glyphnet-core", version = "0.1.0" } +glyphnet-cv = { path = "../glyphnet-cv", version = "0.1.0" } glyphnet-decode = { path = "../glyphnet-decode", version = "0.1.0" } glyphnet-encode = { path = "../glyphnet-encode", version = "0.1.0" } glyphnet-render = { path = "../glyphnet-render", version = "0.1.0" } diff --git a/crates/glyphnet-cli/src/main.rs b/crates/glyphnet-cli/src/main.rs index 0471cdd..234f509 100644 --- a/crates/glyphnet-cli/src/main.rs +++ b/crates/glyphnet-cli/src/main.rs @@ -1,14 +1,18 @@ -use std::{fs, path::PathBuf}; +use std::{env, fs, path::PathBuf}; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand, ValueEnum}; use glyphnet_core::{ EccLevel, ProfileId, SymbolGeometry, TransmissionMode, profile_catalog, profile_spec, }; +use glyphnet_cv::{VisionProfile, adaptive_threshold, grayscale, warp_perspective_gray}; use glyphnet_decode::RasterDecoder; use glyphnet_encode::{Encoder, EncoderConfig}; use glyphnet_render::{RasterRenderer, RenderOptions, SvgRenderer}; -use glyphnet_scanner::{CameraFrame, Scanner, ScannerConfig, scan_still}; +use glyphnet_scanner::{ + CameraFrame, FailedStillScan, Scanner, ScannerConfig, scan_still_robust, + scan_still_with_diagnostics, +}; mod auth; #[derive(Debug, Parser)] @@ -682,8 +686,30 @@ fn scan( ) -> Result<()> { let image = image::open(&input).with_context(|| format!("failed to open image {}", input.display()))?; + let debug = ScanDebug::from_env(&input); + if let Some(debug) = &debug { + debug.dump_base_steps(&image, mode)?; + } let detached_signature = auth::load_detached_verification_input(detached_auth_file.as_ref())?; - let scanned = scan_still(&image, mode).context("failed to scan image")?; + let scanned = match scan_still_with_diagnostics(&image, mode) { + Ok(scanned) => { + if let Some(debug) = &debug { + debug.dump_success(&image, mode, &scanned)?; + } + scanned + } + Err(failed) => { + if let Some(debug) = &debug { + let _ = debug.dump_failure(&image, mode, &failed); + } + let robust = scan_still_robust(&image, mode) + .map_err(|_| anyhow::anyhow!("failed to scan image: {}", failed.error))?; + if let Some(debug) = &debug { + debug.dump_robust_success(&image, mode, &robust)?; + } + robust + } + }; let crop = scanned.crop.map(|region| { serde_json::json!({ "x": region.x, @@ -744,6 +770,326 @@ fn scan( Ok(()) } +struct ScanDebug { + enabled: bool, + dir: PathBuf, +} + +impl ScanDebug { + fn from_env(input: &std::path::Path) -> Option { + if env::var_os("GLYPHNET_SCAN_DEBUG").is_none() { + return None; + } + let input_stem = input + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or("scan"); + let dir = env::var_os("GLYPHNET_SCAN_DEBUG_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target").join("scan-debug")) + .join(input_stem); + Some(Self { enabled: true, dir }) + } + + fn dump_base_steps(&self, image: &image::DynamicImage, mode: TransmissionMode) -> Result<()> { + if !self.enabled { + return Ok(()); + } + fs::create_dir_all(&self.dir) + .with_context(|| format!("failed to create debug directory {}", self.dir.display()))?; + image + .save(self.dir.join("00-input.png")) + .context("failed to save debug input image")?; + let gray = grayscale(image).context("failed to compute debug grayscale image")?; + gray.save(self.dir.join("01-gray.png")) + .context("failed to save debug grayscale image")?; + let profile = VisionProfile::for_mode(mode); + let binary = adaptive_threshold(&gray, profile.threshold_radius, profile.threshold_bias) + .context("failed to compute debug binary image")?; + binary + .save(self.dir.join("02-binary.png")) + .context("failed to save debug binary image")?; + self.write_binary_text(&binary)?; + self.write_binary_grid_image(&binary)?; + Ok(()) + } + + fn dump_success( + &self, + image: &image::DynamicImage, + mode: TransmissionMode, + scanned: &glyphnet_scanner::StillScanResult, + ) -> Result<()> { + self.dump_attempt_regions(image, mode, &scanned.attempts)?; + self.write_decoded_matrix_artifacts(&scanned.decoded.decoded.matrix)?; + if let Some(crop) = scanned.crop { + let cropped = + image::imageops::crop_imm(image, crop.x, crop.y, crop.width, crop.height).to_image(); + cropped + .save(self.dir.join("03-crop-hit.png")) + .context("failed to save debug crop hit image")?; + } + if let (Some(quad), Some((width, height))) = (scanned.quad, scanned.warp_size) { + let gray = grayscale(image).context("failed to compute gray for debug warp")?; + let warped = warp_perspective_gray(&gray, quad, width, height) + .context("failed to compute debug warp image")?; + warped + .save(self.dir.join("04-quad-warp.png")) + .context("failed to save debug warp image")?; + } + let telemetry = scanned.telemetry(); + let payload = serde_json::json!({ + "ok": true, + "mode": mode.to_string(), + "auto": { + "module_px": scanned.decoded.info.module_px, + "quiet_zone_modules": scanned.decoded.info.quiet_zone_modules, + "threshold": scanned.decoded.info.threshold, + "layout": layout_name(scanned.decoded.info.layout) + }, + "crop": scanned.crop.map(|region| serde_json::json!({ + "x": region.x, + "y": region.y, + "width": region.width, + "height": region.height + })), + "quad": scanned.quad.map(|quad| serde_json::json!({ + "top_left": {"x": quad.top_left.x, "y": quad.top_left.y}, + "top_right": {"x": quad.top_right.x, "y": quad.top_right.y}, + "bottom_right": {"x": quad.bottom_right.x, "y": quad.bottom_right.y}, + "bottom_left": {"x": quad.bottom_left.x, "y": quad.bottom_left.y} + })), + "warp_size": scanned.warp_size.map(|(w, h)| serde_json::json!({"width": w, "height": h})), + "attempts": scanned.attempts.iter().enumerate().map(|(index, attempt)| serde_json::json!({ + "index": index, + "detector": attempt.detector, + "layout_hint": attempt.layout_hint.map(layout_name), + "stage": attempt.stage, + "decoded": attempt.decoded, + "error": attempt.error, + "duration_micros": attempt.duration_micros, + "region": {"x": attempt.region.x, "y": attempt.region.y, "width": attempt.region.width, "height": attempt.region.height} + })).collect::>(), + "timings": { + "total_micros": telemetry.timings.total_micros, + "full_frame_micros": telemetry.timings.full_frame_micros, + "grayscale_micros": telemetry.timings.grayscale_micros, + "threshold_micros": telemetry.timings.threshold_micros, + "quad_micros": telemetry.timings.quad_micros, + "candidate_micros": telemetry.timings.candidate_micros, + "decode_attempts_micros": telemetry.timings.decode_attempts_micros + } + }); + fs::write( + self.dir.join("diagnostics.json"), + serde_json::to_string_pretty(&payload)?, + ) + .context("failed to write debug diagnostics json")?; + Ok(()) + } + + fn dump_failure( + &self, + image: &image::DynamicImage, + mode: TransmissionMode, + failed: &FailedStillScan, + ) -> Result<()> { + self.dump_attempt_regions(image, mode, &failed.attempts)?; + let payload = serde_json::json!({ + "ok": false, + "mode": mode.to_string(), + "error": failed.error.to_string(), + "attempts": failed.attempts.iter().enumerate().map(|(index, attempt)| serde_json::json!({ + "index": index, + "detector": attempt.detector, + "layout_hint": attempt.layout_hint.map(layout_name), + "stage": attempt.stage, + "decoded": attempt.decoded, + "error": attempt.error, + "duration_micros": attempt.duration_micros, + "region": {"x": attempt.region.x, "y": attempt.region.y, "width": attempt.region.width, "height": attempt.region.height} + })).collect::>(), + "timings": { + "total_micros": failed.timings.total_micros, + "full_frame_micros": failed.timings.full_frame_micros, + "grayscale_micros": failed.timings.grayscale_micros, + "threshold_micros": failed.timings.threshold_micros, + "quad_micros": failed.timings.quad_micros, + "candidate_micros": failed.timings.candidate_micros, + "decode_attempts_micros": failed.timings.decode_attempts_micros + } + }); + fs::write( + self.dir.join("diagnostics.json"), + serde_json::to_string_pretty(&payload)?, + ) + .context("failed to write failure diagnostics json")?; + Ok(()) + } + + fn dump_robust_success( + &self, + image: &image::DynamicImage, + mode: TransmissionMode, + scanned: &glyphnet_scanner::StillScanResult, + ) -> Result<()> { + self.dump_success(image, mode, scanned)?; + let path = self.dir.join("diagnostics.json"); + let data = fs::read_to_string(&path).context("failed to read diagnostics json")?; + let mut payload: serde_json::Value = + serde_json::from_str(&data).context("failed to parse diagnostics json")?; + payload["used_robust_fallback"] = serde_json::json!(true); + fs::write(path, serde_json::to_string_pretty(&payload)?) + .context("failed to rewrite diagnostics json")?; + Ok(()) + } + + fn dump_attempt_regions( + &self, + image: &image::DynamicImage, + mode: TransmissionMode, + attempts: &[glyphnet_scanner::ScanAttempt], + ) -> Result<()> { + let profile = VisionProfile::for_mode(mode); + for (index, attempt) in attempts.iter().enumerate() { + let region = attempt.region; + let cropped = + image::imageops::crop_imm(image, region.x, region.y, region.width, region.height) + .to_image(); + let state = if attempt.decoded { "hit" } else { "miss" }; + let stem = format!( + "attempt-{index:03}-{state}-{}-{}-{}x{}+{}+{}", + attempt.detector, + attempt.stage, + region.width, + region.height, + region.x, + region.y + ); + cropped + .save(self.dir.join(format!("{stem}-crop.png"))) + .context("failed to save attempt debug crop")?; + + let cropped_dyn = image::DynamicImage::ImageRgba8(cropped); + let gray = grayscale(&cropped_dyn).context("failed to grayscale attempt crop")?; + gray.save(self.dir.join(format!("{stem}-gray.png"))) + .context("failed to save attempt grayscale image")?; + + let binary = adaptive_threshold(&gray, profile.threshold_radius, profile.threshold_bias) + .context("failed to threshold attempt crop")?; + binary + .save(self.dir.join(format!("{stem}-binary.png"))) + .context("failed to save attempt binary image")?; + self.write_binary_text_named(&binary, &format!("{stem}-binary.txt"))?; + self.write_binary_grid_image_named(&binary, &format!("{stem}-binary-grid.png"))?; + } + Ok(()) + } + + fn write_binary_text(&self, binary: &image::GrayImage) -> Result<()> { + self.write_binary_text_named(binary, "02-binary.txt") + } + + fn write_binary_text_named(&self, binary: &image::GrayImage, name: &str) -> Result<()> { + let mut text = + String::with_capacity((binary.width() as usize + 1).saturating_mul(binary.height() as usize)); + for y in 0..binary.height() { + for x in 0..binary.width() { + let pixel = binary.get_pixel(x, y).0[0]; + text.push(if pixel == 0 { '1' } else { '0' }); + } + text.push('\n'); + } + fs::write(self.dir.join(name), text) + .context("failed to write debug binary text file")?; + Ok(()) + } + + fn write_binary_grid_image(&self, binary: &image::GrayImage) -> Result<()> { + self.write_binary_grid_image_named(binary, "02-binary-grid.png") + } + + fn write_binary_grid_image_named(&self, binary: &image::GrayImage, name: &str) -> Result<()> { + const CELL: u32 = 4; + const GRID: u32 = 1; + + let width = binary.width(); + let height = binary.height(); + let out_width = width.saturating_mul(CELL + GRID).saturating_add(GRID); + let out_height = height.saturating_mul(CELL + GRID).saturating_add(GRID); + let mut out = image::GrayImage::from_pixel(out_width, out_height, image::Luma([220])); + + for y in 0..height { + for x in 0..width { + let pixel = binary.get_pixel(x, y).0[0]; + let start_x = GRID + x * (CELL + GRID); + let start_y = GRID + y * (CELL + GRID); + for dy in 0..CELL { + for dx in 0..CELL { + out.put_pixel(start_x + dx, start_y + dy, image::Luma([pixel])); + } + } + } + } + + out.save(self.dir.join(name)) + .context("failed to save debug binary grid image")?; + Ok(()) + } + + fn write_decoded_matrix_artifacts(&self, matrix: &glyphnet_core::SymbolMatrix) -> Result<()> { + let width = usize::from(matrix.width()); + let height = usize::from(matrix.height()); + + let mut text = String::with_capacity((width + 1).saturating_mul(height)); + for y in 0..matrix.height() { + for x in 0..matrix.width() { + let cell = matrix.get(x, y).map_err(|err| anyhow::anyhow!(err.to_string()))?; + text.push(if cell.is_dark() { '1' } else { '0' }); + } + text.push('\n'); + } + fs::write(self.dir.join("05-decoded-matrix.txt"), text) + .context("failed to write decoded matrix text")?; + + const CELL: u32 = 14; + const GRID: u32 = 1; + let out_width = u32::from(matrix.width()) + .saturating_mul(CELL + GRID) + .saturating_add(GRID); + let out_height = u32::from(matrix.height()) + .saturating_mul(CELL + GRID) + .saturating_add(GRID); + let mut out = image::GrayImage::from_pixel(out_width, out_height, image::Luma([210])); + + for y in 0..matrix.height() { + for x in 0..matrix.width() { + let cell = matrix.get(x, y).map_err(|err| anyhow::anyhow!(err.to_string()))?; + let value = if cell.is_dark() { 0 } else { 255 }; + let start_x = GRID + u32::from(x) * (CELL + GRID); + let start_y = GRID + u32::from(y) * (CELL + GRID); + for dy in 0..CELL { + for dx in 0..CELL { + out.put_pixel(start_x + dx, start_y + dy, image::Luma([value])); + } + } + } + } + out.save(self.dir.join("05-decoded-matrix-grid.png")) + .context("failed to save decoded matrix grid image")?; + + let data_bits = matrix.read_data_bits(); + let mut data_bits_text = String::with_capacity(data_bits.len() + 1); + for bit in data_bits { + data_bits_text.push(if bit { '1' } else { '0' }); + } + data_bits_text.push('\n'); + fs::write(self.dir.join("05-decoded-data-bits.txt"), data_bits_text) + .context("failed to write decoded data bits text")?; + Ok(()) + } +} + // Auth/keyset logic extracted to `auth` module. fn scan_burst(input_dir: PathBuf, mode: TransmissionMode) -> Result<()> { diff --git a/crates/glyphnet-decode/src/lib.rs b/crates/glyphnet-decode/src/lib.rs index 8911b96..e4ffd2d 100644 --- a/crates/glyphnet-decode/src/lib.rs +++ b/crates/glyphnet-decode/src/lib.rs @@ -102,6 +102,23 @@ pub fn decode_matrix(matrix: &SymbolMatrix) -> Result { decode_matrix_with_suspects(matrix, &[]) } +/// Decode a matrix while prioritizing likely-corrupted byte indexes for ECC recovery. +pub fn decode_matrix_with_suspect_bytes( + matrix: &SymbolMatrix, + suspect_bytes: &[usize], +) -> Result { + decode_matrix_with_suspects(matrix, suspect_bytes) +} + +/// Rank likely-corrupted byte indexes from per-data-bit confidence scores. +pub fn suspect_bytes_from_confidence( + matrix: &SymbolMatrix, + bit_confidence: &[u8], + limit: usize, +) -> Vec { + suspect_bytes_from_bit_confidence(matrix, bit_confidence, limit) +} + /// Raster image decoder. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RasterDecoder { diff --git a/crates/glyphnet-decode/src/recovery.rs b/crates/glyphnet-decode/src/recovery.rs index 40aa0e1..2a5cfc9 100644 --- a/crates/glyphnet-decode/src/recovery.rs +++ b/crates/glyphnet-decode/src/recovery.rs @@ -1,4 +1,7 @@ -use glyphnet_core::{Frame, FrameHeader, HEADER_LEN, SymbolMatrix, bitstream}; +use glyphnet_core::{ + EccLevel, Frame, FrameHeader, GlyphError, HEADER_LEN, MAGIC, SymbolMatrix, TransmissionMode, + WIRE_VERSION, bitstream, +}; use glyphnet_ecc::{ RecoveryMethod, RecoveryTelemetry, try_recover_for_mode_with_suspects_and_telemetry, verify_for_mode, @@ -13,8 +16,16 @@ pub(crate) fn decode_matrix_with_suspects( suspect_bytes: &[usize], ) -> Result { let bits = matrix.read_data_bits(); - let sampled_bytes = bitstream::bits_to_bytes(&bits); - let header = FrameHeader::decode(&sampled_bytes)?; + let mut sampled_bytes = bitstream::bits_to_bytes(&bits); + let header = match FrameHeader::decode(&sampled_bytes) { + Ok(header) => header, + Err(GlyphError::HeaderChecksumMismatch) => { + let header = decode_header_fields_without_crc(&sampled_bytes)?; + sampled_bytes[28..32].copy_from_slice(&header.header_crc.to_be_bytes()); + header + } + Err(error) => return Err(error.into()), + }; let data_len = HEADER_LEN + header.payload_len as usize; if verify_for_mode(header.mode, header.ecc_level, &sampled_bytes, data_len) { let frame = Frame::decode(&sampled_bytes)?; @@ -60,3 +71,36 @@ pub(crate) fn decode_matrix_with_suspects( Err(DecodeError::EccMismatch) } + +fn decode_header_fields_without_crc(bytes: &[u8]) -> std::result::Result { + if bytes.len() < HEADER_LEN { + return Err(GlyphError::Truncated { + needed: HEADER_LEN, + actual: bytes.len(), + }); + } + if bytes[0..4] != MAGIC { + return Err(GlyphError::InvalidMagic); + } + if bytes[4] != WIRE_VERSION { + return Err(GlyphError::UnsupportedVersion(bytes[4])); + } + let mode = TransmissionMode::from_wire(bytes[5])?; + let ecc_level = EccLevel::from_wire(bytes[6])?; + let frame_index = u16::from_be_bytes([bytes[8], bytes[9]]); + let frame_count = u16::from_be_bytes([bytes[10], bytes[11]]); + let stream_id = u64::from_be_bytes([ + bytes[12], bytes[13], bytes[14], bytes[15], bytes[16], bytes[17], bytes[18], bytes[19], + ]); + let payload_len = u32::from_be_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]); + let payload_crc = u32::from_be_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]); + FrameHeader::new( + mode, + ecc_level, + frame_index, + frame_count, + stream_id, + payload_len, + payload_crc, + ) +} diff --git a/crates/glyphnet-ecc/src/lib.rs b/crates/glyphnet-ecc/src/lib.rs index f78e5a4..1452011 100644 --- a/crates/glyphnet-ecc/src/lib.rs +++ b/crates/glyphnet-ecc/src/lib.rs @@ -602,6 +602,24 @@ pub fn try_recover_for_mode_with_suspects_and_telemetry( } } } + for count in 3..=suspect_pool.len().min(rs.parity_shards) { + attempts += 1; + if attempts > max_attempts { + telemetry.attempts = attempts; + telemetry.max_attempts_exceeded = true; + return (None, telemetry); + } + if let Some(candidate) = + rs.recover_data_shards(&normalized, data_len, &suspect_pool[..count]) + && rs.verify(&candidate, data_len) + && Frame::decode(&candidate).is_ok() + { + telemetry.recovered = true; + telemetry.attempts = attempts; + telemetry.method = RecoveryMethod::ReedSolomonPair; + return (Some(to_encoded_layout(candidate)), telemetry); + } + } } for (index, already_tried) in tried_index.iter().enumerate().take(data_len) { if *already_tried { diff --git a/crates/glyphnet-scanner/Cargo.toml b/crates/glyphnet-scanner/Cargo.toml index 7928565..8a322d0 100644 --- a/crates/glyphnet-scanner/Cargo.toml +++ b/crates/glyphnet-scanner/Cargo.toml @@ -13,6 +13,7 @@ readme.workspace = true default = ["std"] std = [] async = ["tokio"] +opencv-fallback = ["dep:opencv"] [dependencies] glyphnet-core = { path = "../glyphnet-core", version = "0.1.0" } @@ -26,6 +27,7 @@ tracing.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rayon = "1" +opencv = { version = "0.98.2", optional = true, default-features = false, features = ["imgproc"] } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3" diff --git a/crates/glyphnet-scanner/src/candidates.rs b/crates/glyphnet-scanner/src/candidates.rs index caccd9b..3686fd1 100644 --- a/crates/glyphnet-scanner/src/candidates.rs +++ b/crates/glyphnet-scanner/src/candidates.rs @@ -75,10 +75,12 @@ pub(crate) fn still_scan_candidates( } else { 12 }; - let max_generic = if !robust || large_image { - 0 - } else { + let max_generic = if large_image { + if robust { 8 } else { 4 } + } else if robust { MAX_GENERIC_CANDIDATES + } else { + 4 }; let max_dark_bounds = if !robust { 1 @@ -88,6 +90,21 @@ pub(crate) fn still_scan_candidates( MAX_DARK_BOUNDS_CANDIDATES }; let mut candidates = Vec::new(); + let precomputed_dark_bounds = dark_bounds(binary); + + let mut border_trim = ribbon_border_trim_candidates(image_width, image_height); + border_trim.truncate(10); + candidates.extend(border_trim); + + let mut roi_group = ribbon_symbol_roi_candidates(binary, image_width, image_height); + roi_group.truncate(8); + candidates.extend(roi_group); + + if let Some(bounds) = precomputed_dark_bounds { + let mut anchored = ribbon_top_left_scale_candidates(bounds, image_width, image_height); + anchored.truncate(10); + candidates.extend(anchored); + } if !large_image && let Some(bounds) = content_bounds(image) { let mut content = @@ -112,7 +129,7 @@ pub(crate) fn still_scan_candidates( if candidates.len() < max_total && should_try_dark_bounds_fallback(image_width, image_height, candidates.len()) - && let Some(bounds) = dark_bounds(binary) + && let Some(bounds) = precomputed_dark_bounds { let mut dark_bounds = ribbon_dark_bounds_candidates(bounds, padding, image_width, image_height); @@ -122,28 +139,249 @@ pub(crate) fn still_scan_candidates( } if candidates.len() < max_total { - let mut fallback = coarse_ribbon_grid_candidates(image_width, image_height); + let mut fallback = coarse_ribbon_grid_candidates(binary, image_width, image_height); fallback.truncate(max_total - candidates.len()); candidates.extend(fallback); } + refine_candidate_priority(&mut candidates, image_width, image_height); candidates.truncate(max_total); candidates } -fn coarse_ribbon_grid_candidates(image_width: u32, image_height: u32) -> Vec { +fn refine_candidate_priority( + candidates: &mut Vec, + image_width: u32, + image_height: u32, +) { + let image_area = u64::from(image_width) + .saturating_mul(u64::from(image_height)) + .max(1); + candidates.retain(|candidate| { + let region = candidate.region; + let area = u64::from(region.width).saturating_mul(u64::from(region.height)); + let area_pct = area.saturating_mul(100) / image_area; + let top_edge = region.y <= (image_height / 32).max(8); + let is_large = area_pct >= 60; + + let reject_signature_window = candidate.detector == CandidateDetector::RibbonWeave + && candidate.stage == "signature-window" + && top_edge + && is_large; + let reject_full_band = candidate.detector == CandidateDetector::GenericBinary + && candidate.stage == "horizontal-band" + && area_pct >= 80; + let reject_full_aspect = candidate.detector == CandidateDetector::GenericBinary + && candidate.stage == "horizontal-aspect" + && area_pct >= 80; + !(reject_signature_window || reject_full_band || reject_full_aspect) + }); + + let has_non_top_ribbon = candidates.iter().any(|candidate| { + candidate.detector == CandidateDetector::RibbonWeave + && !is_top_heavy_ribbon_region(candidate.region, image_height) + }); + if has_non_top_ribbon { + candidates.retain(|candidate| { + candidate.detector != CandidateDetector::RibbonWeave + || !is_top_heavy_ribbon_region(candidate.region, image_height) + }); + } + + candidates.sort_by_key(|candidate| candidate_priority_score(*candidate, image_width, image_height)); +} + +fn is_top_heavy_ribbon_region(region: ScanRegion, image_height: u32) -> bool { + let near_top = region.y <= (image_height / 32).max(8); + let very_tall = region.height >= (image_height.saturating_mul(3) / 5).max(160); + near_top && very_tall +} + +fn candidate_priority_score( + candidate: ScanCandidate, + image_width: u32, + image_height: u32, +) -> u64 { + let region = candidate.region; + let mut score = 0u64; + + if candidate.detector == CandidateDetector::RibbonWeave { + if candidate.stage == "border-trim" { + score = score.saturating_sub(140); + } + if candidate.stage == "roi-group" { + score = score.saturating_sub(80); + } + let expected_ratio = 104.0f32 / 44.0f32; + let ratio = region.width as f32 / region.height.max(1) as f32; + score = score.saturating_add(((ratio - expected_ratio).abs() * 100.0).round() as u64); + } + + if region.y <= (image_height / 32).max(8) { + score = score.saturating_add(120); + } + if region.x == 0 + || region.y == 0 + || region.x.saturating_add(region.width) >= image_width + || region.y.saturating_add(region.height) >= image_height + { + score = score.saturating_add(60); + } + + let area = u64::from(region.width).saturating_mul(u64::from(region.height)); + let image_area = u64::from(image_width) + .saturating_mul(u64::from(image_height)) + .max(1); + let area_pct = area.saturating_mul(100) / image_area; + if area_pct > 35 { + score = score.saturating_add((area_pct - 35) * 4); + } + if candidate.detector == CandidateDetector::GenericBinary && candidate.stage == "horizontal-aspect" { + score = score.saturating_add(260); + } + score +} + +fn ribbon_border_trim_candidates(image_width: u32, image_height: u32) -> Vec { + if image_width < 220 || image_height < 90 { + return Vec::new(); + } + let mut out = Vec::new(); + for pct in [2u32, 3, 4, 5, 6, 8, 10] { + let trim_x = image_width.saturating_mul(pct) / 100; + let trim_y = image_height.saturating_mul(pct) / 100; + let width = image_width.saturating_sub(trim_x.saturating_mul(2)); + let height = image_height.saturating_sub(trim_y.saturating_mul(2)); + if width < 160 || height < 60 { + continue; + } + push_unique_candidate( + &mut out, + CandidateDetector::RibbonWeave, + Some(LayoutFamily::RibbonWeave), + "border-trim", + ScanRegion { + x: trim_x, + y: trim_y, + width, + height, + }, + ); + } + out +} + +fn ribbon_symbol_roi_candidates( + binary: &GrayImage, + image_width: u32, + image_height: u32, +) -> Vec { + let mut seeds: Vec = dark_components(binary) + .into_iter() + .filter(|c| { + let b = c.bounds; + if c.pixels < 60 || b.width < 20 || b.height < 10 { + return false; + } + let aspect = b.width as f32 / b.height.max(1) as f32; + (0.6..=12.0).contains(&aspect) + }) + .take(120) + .collect(); + if seeds.is_empty() { + return Vec::new(); + } + + #[derive(Clone, Copy)] + struct Group { + bounds: ScanRegion, + pixels: u32, + score: i64, + } + let mut groups = Vec::new(); + for seed in &seeds { + let mut bounds = seed.bounds; + let mut pixels = seed.pixels; + let sx = seed.bounds.x + seed.bounds.width / 2; + let sy = seed.bounds.y + seed.bounds.height / 2; + for other in &seeds { + let ox = other.bounds.x + other.bounds.width / 2; + let oy = other.bounds.y + other.bounds.height / 2; + let dx = sx.abs_diff(ox); + let dy = sy.abs_diff(oy); + if dx <= seed.bounds.width.saturating_mul(4) && dy <= seed.bounds.height.saturating_mul(3) { + bounds = union_region(bounds, other.bounds); + pixels = pixels.saturating_add(other.pixels); + } + } + let area = bounds.width.saturating_mul(bounds.height).max(1); + let area_ratio = area as f32 / (image_width.saturating_mul(image_height).max(1) as f32); + if !(0.01..=0.85).contains(&area_ratio) { + continue; + } + let aspect = bounds.width as f32 / bounds.height.max(1) as f32; + let aspect_penalty = ((aspect - (104.0 / 44.0)).abs() * 100.0) as i64; + let top_penalty = if bounds.y < image_height / 12 { 120 } else { 0 }; + let score = pixels as i64 - aspect_penalty - top_penalty; + groups.push(Group { bounds, pixels, score }); + } + groups.sort_by(|a, b| b.score.cmp(&a.score).then(b.pixels.cmp(&a.pixels))); + groups.truncate(4); + + let mut out = Vec::new(); + for g in groups { + for w_scale in [1.10f32, 1.25, 1.40, 1.55] { + let w = ((g.bounds.width as f32) * w_scale).round() as u32; + if w < 120 || w > image_width { + continue; + } + if w > image_width.saturating_mul(80) / 100 { + continue; + } + let nominal_h = ((w as f32) * (44.0 / 104.0)).round() as u32; + for h_scale in [1.00f32, 1.15, 1.30] { + let h = ((nominal_h as f32) * h_scale).round() as u32; + if h < 70 || h > image_height { + continue; + } + let cx = g.bounds.x + g.bounds.width / 2; + let cy = g.bounds.y + g.bounds.height / 2; + let x = cx.saturating_sub(w / 2).min(image_width.saturating_sub(w)); + let y = cy.saturating_sub(h / 2).min(image_height.saturating_sub(h)); + let region = ScanRegion { x, y, width: w, height: h }; + if !region_contains(region, g.bounds) { + continue; + } + push_unique_candidate( + &mut out, + CandidateDetector::RibbonWeave, + Some(LayoutFamily::RibbonWeave), + "roi-group", + region, + ); + } + } + } + out +} + +fn coarse_ribbon_grid_candidates( + binary: &GrayImage, + image_width: u32, + image_height: u32, +) -> Vec { if image_width < 240 || image_height < 120 { return Vec::new(); } let mut candidates = Vec::new(); - for module_px in [4u32, 5, 3, 6, 7, 8] { + for module_px in [4u32, 5, 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] { let width = 104 * module_px; let height = 44 * module_px; if width > image_width || height > image_height { continue; } - let x_fracs = [0.06_f32, 0.115, 0.16, 0.22, 0.30, 0.38]; - let y_fracs = [0.08_f32, 0.14, 0.20, 0.233, 0.28, 0.32]; + let x_fracs = [0.04_f32, 0.10, 0.16, 0.24, 0.32, 0.42, 0.52, 0.62, 0.72, 0.80]; + let y_fracs = [0.07_f32, 0.12, 0.18, 0.24, 0.30, 0.36, 0.44]; for xf in x_fracs { for yf in y_fracs { let x = ((image_width as f32 * xf).round() as u32) @@ -167,6 +405,29 @@ fn coarse_ribbon_grid_candidates(image_width: u32, image_height: u32) -> Vec Vec Vec u32 { + let x_end = region.x.saturating_add(region.width).min(binary.width()); + let y_end = region.y.saturating_add(region.height).min(binary.height()); + let mut dark = 0u32; + for y in region.y..y_end { + for x in region.x..x_end { + if binary.get_pixel(x, y).0[0] == 0 { + dark = dark.saturating_add(1); + } + } + } + dark +} + fn content_bounds(image: &DynamicImage) -> Option { let rgba = image.to_rgba8(); let mut min_x = image.width(); @@ -346,6 +623,11 @@ fn ribbon_dark_bounds_candidates( region, )); } + candidates.extend(ribbon_top_left_scale_candidates( + bounds, + image_width, + image_height, + )); let expanded = expand_region(bounds, padding, image_width, image_height); candidates.push(ScanCandidate::new( CandidateDetector::RibbonWeave, @@ -364,6 +646,77 @@ fn ribbon_dark_bounds_candidates( candidates } +fn ribbon_top_left_scale_candidates( + bounds: ScanRegion, + image_width: u32, + image_height: u32, +) -> Vec { + let mut candidates = Vec::new(); + if bounds.width < 64 || bounds.height < 24 { + return candidates; + } + + let est_module_w = (bounds.width as f32 / 96.0).round() as i32; + let est_module_h = (bounds.height as f32 / 36.0).round() as i32; + let est_module = ((est_module_w + est_module_h) / 2).clamp(2, 16) as u32; + let mut module_values = Vec::new(); + for delta in -5..=6 { + let value = (est_module as i32 + delta).clamp(2, 18) as u32; + if !module_values.contains(&value) { + module_values.push(value); + } + } + + for module_px in module_values { + let width = 104u32.saturating_mul(module_px); + let height = 44u32.saturating_mul(module_px); + if width > image_width || height > image_height { + continue; + } + for margin_mul in [4u32, 6, 8, 10] { + let quiet = margin_mul.saturating_mul(module_px); + let x = bounds.x.saturating_sub(quiet); + let y = bounds.y.saturating_sub(quiet); + let region = clamp_region( + ScanRegion { + x, + y, + width, + height, + }, + image_width, + image_height, + ); + if !plausible_region(region) { + continue; + } + // Keep only candidates that still contain the detected dark content bounds. + if !region_contains(region, bounds) { + continue; + } + push_unique_candidate( + &mut candidates, + CandidateDetector::RibbonWeave, + Some(LayoutFamily::RibbonWeave), + "top-left-scale", + region, + ); + } + } + candidates +} + +fn region_contains(outer: ScanRegion, inner: ScanRegion) -> bool { + let outer_right = outer.x.saturating_add(outer.width); + let outer_bottom = outer.y.saturating_add(outer.height); + let inner_right = inner.x.saturating_add(inner.width); + let inner_bottom = inner.y.saturating_add(inner.height); + inner.x >= outer.x + && inner.y >= outer.y + && inner_right <= outer_right + && inner_bottom <= outer_bottom +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct MatrixFinder { x: u32, @@ -923,7 +1276,9 @@ fn generic_binary_candidates( continue; } let expanded = expand_region(component.bounds, padding * 2, image_width, image_height); - if plausible_region(expanded) { + if plausible_region(expanded) + && plausible_symbol_roi(expanded, image_width, image_height) + { push_unique_candidate( &mut regions, CandidateDetector::GenericBinary, @@ -938,6 +1293,22 @@ fn generic_binary_candidates( regions } +fn plausible_symbol_roi(region: ScanRegion, image_width: u32, image_height: u32) -> bool { + let area = u64::from(region.width).saturating_mul(u64::from(region.height)); + let image_area = u64::from(image_width) + .saturating_mul(u64::from(image_height)) + .max(1); + let area_pct = area.saturating_mul(100) / image_area; + if area_pct < 1 || area_pct > 45 { + return false; + } + if region.height < 120 || region.width < 260 { + return false; + } + let aspect = region.width as f32 / region.height.max(1) as f32; + (1.8..=3.8).contains(&aspect) +} + #[derive(Debug, Clone, Copy)] struct RailRow { y: u32, diff --git a/crates/glyphnet-scanner/src/decode_paths.rs b/crates/glyphnet-scanner/src/decode_paths.rs index cdcc673..b193da3 100644 --- a/crates/glyphnet-scanner/src/decode_paths.rs +++ b/crates/glyphnet-scanner/src/decode_paths.rs @@ -1,10 +1,15 @@ use glyphnet_core::{Cell, FrameHeader, HEADER_LEN, LayoutFamily, SymbolMatrix, bitstream, layout}; +use glyphnet_cv::{Point, Quad, warp_perspective_gray}; use glyphnet_decode::{ - AutoDecodedSymbol, DecodeError, DecodeOptions, RasterDecoder, decode_matrix, + AutoDecodedSymbol, DecodeError, DecodeOptions, RasterDecoder, decode_matrix_with_suspect_bytes, }; -use image::{DynamicImage, GrayImage}; +use image::{DynamicImage, GrayImage, Luma, Rgba, RgbaImage}; +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::ScanRegion; +static NORMALIZED_DEBUG_COUNTER: AtomicUsize = AtomicUsize::new(0); pub(crate) fn decode_candidate( decoder: &RasterDecoder, @@ -17,11 +22,23 @@ pub(crate) fn decode_candidate( { return Ok(decoded); } - if matches!(candidate.stage, "signature-window" | "coarse-grid") { + if matches!( + candidate.stage, + "signature-window" | "coarse-grid" | "top-left-scale" | "roi-group" | "border-trim" + ) { + let region_area = region.width.saturating_mul(region.height); if let Ok(decoded) = decode_exact_ribbon_candidate(image, region) { return Ok(decoded); } - if let Ok(decoded) = decode_fractional_ribbon_candidate(image) { + if let Ok(decoded) = decode_forced_normalized_ribbon_candidate(image, region) { + return Ok(decoded); + } + if region_area <= 700_000 + && let Ok(decoded) = decode_fractional_ribbon_candidate(image) + { + return Ok(decoded); + } + if let Ok(decoded) = decode_with_padding_variants(decoder, image) { return Ok(decoded); } let target_module_px = 4; @@ -41,6 +58,9 @@ pub(crate) fn decode_candidate( if let Ok(decoded) = decode_exact_ribbon_candidate(&resized, normalized_region) { return Ok(decoded); } + if let Ok(decoded) = decode_with_padding_variants(decoder, &resized) { + return Ok(decoded); + } return Err(DecodeError::AutoDetectFailed); } @@ -54,8 +74,49 @@ pub(crate) fn decode_candidate( if let Ok(decoded) = decode_fractional_ribbon_candidate(image) { return Ok(decoded); } + if let Ok(decoded) = decode_with_padding_variants(decoder, image) { + return Ok(decoded); + } + } + decoder + .decode_auto_with_info(image) + .or_else(|_| decode_with_padding_variants(decoder, image)) +} + +fn decode_with_padding_variants( + decoder: &RasterDecoder, + image: &DynamicImage, +) -> std::result::Result { + for pad_pct in [4u32, 8, 12, 16] { + let padded = white_pad_image(image, pad_pct); + if let Ok(decoded) = decoder.decode_auto_with_info(&padded) { + return Ok(decoded); + } + if let Ok(decoded) = decode_fractional_ribbon_candidate(&padded) { + return Ok(decoded); + } + let region = crate::ScanRegion { + x: 0, + y: 0, + width: padded.width(), + height: padded.height(), + }; + if let Ok(decoded) = decode_exact_ribbon_candidate(&padded, region) { + return Ok(decoded); + } } - decoder.decode_auto_with_info(image) + Err(DecodeError::AutoDetectFailed) +} + +fn white_pad_image(image: &DynamicImage, pad_pct: u32) -> DynamicImage { + let rgba = image.to_rgba8(); + let pad_x = (rgba.width().saturating_mul(pad_pct) / 100).max(2); + let pad_y = (rgba.height().saturating_mul(pad_pct) / 100).max(2); + let out_w = rgba.width().saturating_add(pad_x.saturating_mul(2)); + let out_h = rgba.height().saturating_add(pad_y.saturating_mul(2)); + let mut canvas = RgbaImage::from_pixel(out_w, out_h, Rgba([255, 255, 255, 255])); + image::imageops::overlay(&mut canvas, &rgba, i64::from(pad_x), i64::from(pad_y)); + DynamicImage::ImageRgba8(canvas) } fn decode_exact_matrix_candidate( @@ -120,23 +181,25 @@ fn decode_exact_ribbon_candidate( if region.width >= 104 && region.height >= 44 { let module_px = (region.width / 104).max(1); if region.width == 104 * module_px && region.height == 44 * module_px { - for threshold in [160, 192, 224] { - let exact = RasterDecoder::new(DecodeOptions { - module_px, - quiet_zone_modules: 4, - threshold, - layout: LayoutFamily::RibbonWeave, - }); - if let Ok(decoded) = exact.decode(image) { - return Ok(AutoDecodedSymbol { - decoded, - info: glyphnet_decode::AutoDecodeInfo { - module_px, - quiet_zone_modules: 4, - threshold, - layout: LayoutFamily::RibbonWeave, - }, + for quiet_zone_modules in [0u32, 1, 2, 3, 4, 5, 6, 7, 8] { + for threshold in [144, 160, 176, 192, 208, 224] { + let exact = RasterDecoder::new(DecodeOptions { + module_px, + quiet_zone_modules, + threshold, + layout: LayoutFamily::RibbonWeave, }); + if let Ok(decoded) = exact.decode(image) { + return Ok(AutoDecodedSymbol { + decoded, + info: glyphnet_decode::AutoDecodeInfo { + module_px, + quiet_zone_modules, + threshold, + layout: LayoutFamily::RibbonWeave, + }, + }); + } } } } @@ -169,6 +232,10 @@ fn decode_fractional_ribbon_candidate( thresholds.sort_unstable(); thresholds.dedup(); + if let Ok(decoded) = decode_normalized_dark_bounds_ribbon(&luma, otsu) { + return Ok(decoded); + } + for scale_adjust in [1.0_f32, 0.985, 1.015, 0.97, 1.03] { let scale_x = base_scale_x * scale_adjust; let scale_y = base_scale_y * scale_adjust; @@ -212,10 +279,285 @@ fn decode_fractional_ribbon_candidate( Err(DecodeError::AutoDetectFailed) } +fn decode_forced_normalized_ribbon_candidate( + image: &DynamicImage, + region: ScanRegion, +) -> std::result::Result { + const SYMBOL_WIDTH: u32 = 96; + const SYMBOL_HEIGHT: u32 = 36; + const QUIET: u32 = 4; + + let x = region.x.min(image.width().saturating_sub(1)); + let y = region.y.min(image.height().saturating_sub(1)); + let max_w = image.width().saturating_sub(x); + let max_h = image.height().saturating_sub(y); + let w = region.width.min(max_w); + let h = region.height.min(max_h); + if w < 80 || h < 30 { + return Err(DecodeError::AutoDetectFailed); + } + + let full = image.to_luma8(); + let trims = [0.00f32, 0.06]; + let shifts = [0.00f32, -0.04, 0.04]; + + let est_w = (w / 104).max(2); + let est_h = (h / 44).max(2); + let est = ((est_w + est_h) / 2).clamp(2, 24); + let mut module_candidates = Vec::new(); + for d in -3i32..=3 { + let m = (est as i32 + d).clamp(2, 24) as u32; + if !module_candidates.contains(&m) { + module_candidates.push(m); + } + } + + for trim in trims { + let trim_x = ((w as f32) * trim).round() as u32; + let trim_y = ((h as f32) * trim).round() as u32; + let base_w = w.saturating_sub(trim_x.saturating_mul(2)); + let base_h = h.saturating_sub(trim_y.saturating_mul(2)); + if base_w < 60 || base_h < 24 { + continue; + } + let base_x = x.saturating_add(trim_x); + let base_y = y.saturating_add(trim_y); + + for sx in shifts { + for sy in shifts { + let dx = ((base_w as f32) * sx).round() as i32; + let dy = ((base_h as f32) * sy).round() as i32; + let rx = (base_x as i32 + dx).max(0) as u32; + let ry = (base_y as i32 + dy).max(0) as u32; + let rw = base_w.min(full.width().saturating_sub(rx)); + let rh = base_h.min(full.height().saturating_sub(ry)); + if rw < 60 || rh < 24 { + continue; + } + + let sub = image::imageops::crop_imm(&full, rx, ry, rw, rh).to_image(); + for module_px in &module_candidates { + let total_w = (SYMBOL_WIDTH + QUIET * 2) * *module_px; + let total_h = (SYMBOL_HEIGHT + QUIET * 2) * *module_px; + let content_w = SYMBOL_WIDTH * *module_px; + let content_h = SYMBOL_HEIGHT * *module_px; + let resized = image::imageops::resize( + &sub, + content_w, + content_h, + image::imageops::FilterType::CatmullRom, + ); + + let mut thresholds = vec![fractional_threshold(&resized), 120, 136, 152]; + thresholds.sort_unstable(); + thresholds.dedup(); + + for threshold in thresholds { + let mut bin = resized.clone(); + for px in bin.pixels_mut() { + px.0[0] = if px.0[0] < threshold { 0 } else { 255 }; + } + dump_normalized_debug(region, threshold, &resized, &bin); + let mut canvas = GrayImage::from_pixel(total_w, total_h, Luma([255])); + image::imageops::overlay( + &mut canvas, + &bin, + i64::from(QUIET * *module_px), + i64::from(QUIET * *module_px), + ); + let candidate = DynamicImage::ImageLuma8(canvas); + let full_region = ScanRegion { + x: 0, + y: 0, + width: total_w, + height: total_h, + }; + if let Ok(decoded) = decode_exact_ribbon_candidate(&candidate, full_region) { + return Ok(decoded); + } + let auto = RasterDecoder::default(); + if let Ok(decoded) = auto.decode_auto_with_info(&candidate) { + return Ok(decoded); + } + } + } + } + } + } + + Err(DecodeError::AutoDetectFailed) +} + +fn dump_normalized_debug(region: ScanRegion, threshold: u8, resized: &GrayImage, bin: &GrayImage) { + if std::env::var_os("GLYPHNET_SCAN_DEBUG").is_none() { + return; + } + let base_dir = std::env::var_os("GLYPHNET_SCAN_DEBUG_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target/scan-debug")); + let out_dir = base_dir.join("normalized"); + if fs::create_dir_all(&out_dir).is_err() { + return; + } + let idx = NORMALIZED_DEBUG_COUNTER.fetch_add(1, Ordering::Relaxed); + let prefix = format!( + "n{:04}-{}x{}+{}+{}-t{}", + idx, region.width, region.height, region.x, region.y, threshold + ); + let _ = resized.save(out_dir.join(format!("{prefix}-normalized-resized.png"))); + let _ = bin.save(out_dir.join(format!("{prefix}-normalized-binary.png"))); + let mut txt = String::new(); + for y in 0..bin.height() { + for x in 0..bin.width() { + let value = if bin.get_pixel(x, y).0[0] == 0 { '1' } else { '0' }; + txt.push(value); + } + txt.push('\n'); + } + let _ = fs::write( + out_dir.join(format!("{prefix}-normalized-binary.txt")), + txt, + ); +} + fn module_shifts(radius: i32) -> impl Iterator { (-radius * 2..=radius * 2).map(|value| value as f32 * 0.5) } +fn decode_normalized_dark_bounds_ribbon( + luma: &GrayImage, + otsu: u8, +) -> std::result::Result { + const MODULE_PX: u32 = 4; + const SYMBOL_WIDTH: u32 = 96; + const SYMBOL_HEIGHT: u32 = 36; + const QUIET_MODULES: u32 = 4; + const TOTAL_WIDTH: u32 = (SYMBOL_WIDTH + QUIET_MODULES * 2) * MODULE_PX; + const TOTAL_HEIGHT: u32 = (SYMBOL_HEIGHT + QUIET_MODULES * 2) * MODULE_PX; + + for threshold in [otsu.saturating_sub(80).clamp(40, 72), 48, 56] { + let Some(bounds) = dark_bounds_luma(luma, threshold) else { + continue; + }; + let (min_x, min_y, max_x, max_y) = bounds; + let width = max_x.saturating_sub(min_x).saturating_add(1); + let height = max_y.saturating_sub(min_y).saturating_add(1); + if width < SYMBOL_WIDTH || height < SYMBOL_HEIGHT { + continue; + } + let Some(quad) = dark_content_quad(luma, threshold, min_x, max_x) else { + continue; + }; + let mut normalized = warp_perspective_gray( + luma, + quad, + SYMBOL_WIDTH * MODULE_PX, + SYMBOL_HEIGHT * MODULE_PX, + ) + .map_err(|_| DecodeError::AutoDetectFailed)?; + let normalized_threshold = fractional_threshold(&normalized).clamp(96, 160); + for pixel in normalized.pixels_mut() { + pixel.0[0] = if pixel.0[0] < normalized_threshold { + 0 + } else { + 255 + }; + } + let mut canvas = GrayImage::from_pixel(TOTAL_WIDTH, TOTAL_HEIGHT, Luma([255])); + image::imageops::overlay( + &mut canvas, + &normalized, + i64::from(QUIET_MODULES * MODULE_PX), + i64::from(QUIET_MODULES * MODULE_PX), + ); + let image = DynamicImage::ImageLuma8(canvas); + let region = ScanRegion { + x: 0, + y: 0, + width: TOTAL_WIDTH, + height: TOTAL_HEIGHT, + }; + if let Ok(decoded) = decode_exact_ribbon_candidate(&image, region) { + return Ok(decoded); + } + } + Err(DecodeError::AutoDetectFailed) +} + +fn dark_content_quad(luma: &GrayImage, threshold: u8, min_x: u32, max_x: u32) -> Option { + let span = max_x.saturating_sub(min_x).saturating_add(1); + let band = (span / 5).max(16); + let left = vertical_dark_extent(luma, threshold, min_x, min_x.saturating_add(band))?; + let right = vertical_dark_extent(luma, threshold, max_x.saturating_sub(band), max_x)?; + let expand_edge = |top: u32, bottom: u32| { + let module = bottom.saturating_sub(top).max(1) as f32 / 29.0; + ( + (top as f32 - module * 3.0).max(0.0), + (bottom as f32 + module * 4.0).min(luma.height().saturating_sub(1) as f32), + ) + }; + let left = expand_edge(left.0, left.1); + let right = expand_edge(right.0, right.1); + Some(Quad { + top_left: Point { + x: min_x as f32, + y: left.0, + }, + top_right: Point { + x: max_x as f32, + y: right.0, + }, + bottom_right: Point { + x: max_x as f32, + y: right.1, + }, + bottom_left: Point { + x: min_x as f32, + y: left.1, + }, + }) +} + +fn vertical_dark_extent( + luma: &GrayImage, + threshold: u8, + start_x: u32, + end_x: u32, +) -> Option<(u32, u32)> { + let mut min_y = u32::MAX; + let mut max_y = 0u32; + let mut found = false; + for y in 0..luma.height() { + for x in start_x..=end_x.min(luma.width().saturating_sub(1)) { + if luma.get_pixel(x, y)[0] < threshold { + min_y = min_y.min(y); + max_y = max_y.max(y); + found = true; + } + } + } + found.then_some((min_y, max_y)) +} + +fn dark_bounds_luma(luma: &GrayImage, threshold: u8) -> Option<(u32, u32, u32, u32)> { + let mut min_x = u32::MAX; + let mut min_y = u32::MAX; + let mut max_x = 0u32; + let mut max_y = 0u32; + let mut found = false; + for (x, y, pixel) in luma.enumerate_pixels() { + if pixel[0] >= threshold { + continue; + } + found = true; + min_x = min_x.min(x); + min_y = min_y.min(y); + max_x = max_x.max(x); + max_y = max_y.max(y); + } + found.then_some((min_x, min_y, max_x, max_y)) +} + fn fractional_header_precheck( integral: &IntegralGray, origin_x_modules: f32, @@ -290,6 +632,7 @@ fn decode_fractional_with_params( let mut matrix = SymbolMatrix::with_layout(SYMBOL_WIDTH, SYMBOL_HEIGHT, LayoutFamily::RibbonWeave); + let mut bit_confidence = Vec::new(); for y in 0..SYMBOL_HEIGHT { for x in 0..SYMBOL_WIDTH { if let Some(cell) = layout::function_cell_for( @@ -310,10 +653,13 @@ fn decode_fractional_with_params( scale_y, ); matrix.set(x, y, Cell::Data(avg < threshold))?; + bit_confidence.push(avg.abs_diff(threshold)); } } - let decoded = decode_matrix(&matrix)?; + let suspect_bytes = + glyphnet_decode::suspect_bytes_from_confidence(&matrix, &bit_confidence, 16); + let decoded = decode_matrix_with_suspect_bytes(&matrix, &suspect_bytes)?; Ok(AutoDecodedSymbol { decoded, info: glyphnet_decode::AutoDecodeInfo { diff --git a/crates/glyphnet-scanner/src/lib.rs b/crates/glyphnet-scanner/src/lib.rs index 7324755..9d0ad8b 100644 --- a/crates/glyphnet-scanner/src/lib.rs +++ b/crates/glyphnet-scanner/src/lib.rs @@ -41,6 +41,490 @@ pub use detectors::CandidateDetector; use detectors::ScanCandidate; use rectification::{scan_quad_candidates as build_quad_candidates, should_try_quad_rectification}; pub use types::{FailedStillScan, ScanAttempt, ScanTelemetry, ScanTimings, StillScanResult}; +#[cfg(all(not(target_arch = "wasm32"), feature = "opencv-fallback"))] +mod opencv_fallback { + use glyphnet_decode::{AutoDecodedSymbol, RasterDecoder}; + use image::{DynamicImage, GrayImage, Luma}; + use opencv::{ + core::{self, Mat, Point2f, Rect, Scalar, Size, Vector}, + imgproc, + prelude::*, + }; + + pub(crate) fn try_decode_with_opencv(image: &DynamicImage) -> Option { + let luma = image.to_luma8(); + let Ok(src) = gray_to_mat(&luma) else { + return None; + }; + let mut variants = Vec::new(); + let mut page_src = src.clone(); + if let Some(page) = page_rectified_variant(&src) { + page_src = page; + } + + if let Ok(clahe) = clahe_variant(&page_src) { + variants.push(clahe); + } + if let Ok(adaptive) = adaptive_variant(&page_src) { + variants.push(adaptive); + } + if let Ok(clahe) = clahe_variant(&page_src) + && let Ok(combo) = adaptive_variant(&clahe) + { + variants.push(combo); + } + + let mut roi_variants: Vec = Vec::new(); + if let Some(rois) = detect_symbol_rois(&page_src) { + for roi in rois { + if let Ok(crop) = page_src.roi(roi) + && let Ok(crop_mat) = crop.try_clone() + && let Ok(gray) = mat_to_gray(&crop_mat) + { + let crop_img = DynamicImage::ImageLuma8(gray); + roi_variants.push(crop_img.clone()); + roi_variants.push(white_pad(&crop_img, 8)); + roi_variants.push(white_pad(&crop_img, 14)); + } + } + } + if let Some(warped) = quad_warp_variant(&page_src) { + roi_variants.push(warped.clone()); + roi_variants.push(white_pad(&warped, 8)); + roi_variants.push(white_pad(&warped, 14)); + } + + let decoder = RasterDecoder::default(); + for candidate in roi_variants { + if let Ok(decoded) = decoder.decode_auto_with_info(&candidate) { + return Some(decoded); + } + } + for mat in variants { + let Ok(gray) = mat_to_gray(&mat) else { + continue; + }; + let candidate = DynamicImage::ImageLuma8(gray); + if let Ok(decoded) = decoder.decode_auto_with_info(&candidate) { + return Some(decoded); + } + } + None + } + + fn detect_symbol_rois(src: &Mat) -> Option> { + let mut thresh = Mat::default(); + imgproc::adaptive_threshold( + src, + &mut thresh, + 255.0, + imgproc::ADAPTIVE_THRESH_GAUSSIAN_C, + imgproc::THRESH_BINARY_INV, + 31, + 7.0, + ) + .ok()?; + let kernel = imgproc::get_structuring_element( + imgproc::MORPH_RECT, + core::Size::new(3, 3), + core::Point::new(-1, -1), + ) + .ok()?; + let mut closed = Mat::default(); + imgproc::morphology_ex( + &thresh, + &mut closed, + imgproc::MORPH_CLOSE, + &kernel, + core::Point::new(-1, -1), + 2, + core::BORDER_CONSTANT, + core::Scalar::all(0.0), + ) + .ok()?; + + let mut contours: Vector> = Vector::new(); + imgproc::find_contours( + &closed, + &mut contours, + imgproc::RETR_EXTERNAL, + imgproc::CHAIN_APPROX_SIMPLE, + core::Point::new(0, 0), + ) + .ok()?; + + let w = src.cols().max(1) as f32; + let h = src.rows().max(1) as f32; + let img_area = w * h; + + let mut scored: Vec<(f32, Rect)> = Vec::new(); + for contour in contours { + let rect = imgproc::bounding_rect(&contour).ok()?; + if rect.width <= 0 || rect.height <= 0 { + continue; + } + let area = (rect.width * rect.height) as f32; + let area_ratio = area / img_area; + if !(0.001..=0.90).contains(&area_ratio) { + continue; + } + let aspect = rect.width as f32 / rect.height as f32; + if !(0.8..=14.0).contains(&aspect) { + continue; + } + let top_penalty = if rect.y < (h * 0.03) as i32 { 0.15 } else { 0.0 }; + let width_ratio = rect.width as f32 / w; + let height_ratio = rect.height as f32 / h; + let mut score = 0.0f32; + score += (aspect / 3.0).min(1.0) * 0.45; + score += (area_ratio / 0.15).min(1.0) * 0.35; + score += (width_ratio.min(0.8) / 0.8) * 0.2; + score -= top_penalty; + if height_ratio > 0.70 { + score -= 0.10; + } + scored.push((score, rect)); + } + + scored.sort_by(|a, b| b.0.total_cmp(&a.0)); + if scored.is_empty() { + return None; + } + + let mut out = Vec::new(); + for (_, rect) in scored.into_iter().take(12) { + out.push(expand_rect(rect, src.cols(), src.rows(), 0.20)); + out.push(expand_rect(rect, src.cols(), src.rows(), 0.35)); + } + Some(out) + } + + fn expand_rect(rect: Rect, max_w: i32, max_h: i32, ratio: f32) -> Rect { + let dx = ((rect.width as f32) * ratio).round() as i32; + let dy = ((rect.height as f32) * ratio).round() as i32; + let x = (rect.x - dx).max(0); + let y = (rect.y - dy).max(0); + let r = (rect.x + rect.width + dx).min(max_w); + let b = (rect.y + rect.height + dy).min(max_h); + Rect::new(x, y, (r - x).max(1), (b - y).max(1)) + } + + fn white_pad(image: &DynamicImage, pct: u32) -> DynamicImage { + let luma = image.to_luma8(); + let pad_x = ((luma.width() as f32) * (pct as f32 / 100.0)).round() as u32; + let pad_y = ((luma.height() as f32) * (pct as f32 / 100.0)).round() as u32; + let out_w = luma.width().saturating_add(pad_x.saturating_mul(2)); + let out_h = luma.height().saturating_add(pad_y.saturating_mul(2)); + let mut out = GrayImage::from_pixel(out_w.max(1), out_h.max(1), Luma([255])); + for y in 0..luma.height() { + for x in 0..luma.width() { + out.put_pixel(x + pad_x, y + pad_y, *luma.get_pixel(x, y)); + } + } + DynamicImage::ImageLuma8(out) + } + + fn page_rectified_variant(src: &Mat) -> Option { + let mut blur = Mat::default(); + imgproc::gaussian_blur( + src, + &mut blur, + Size::new(5, 5), + 0.0, + 0.0, + core::BORDER_DEFAULT, + core::AlgorithmHint::ALGO_HINT_DEFAULT, + ) + .ok()?; + let mut edges = Mat::default(); + imgproc::canny(&blur, &mut edges, 60.0, 180.0, 3, false).ok()?; + let kernel = imgproc::get_structuring_element( + imgproc::MORPH_RECT, + Size::new(5, 5), + core::Point::new(-1, -1), + ) + .ok()?; + let mut closed = Mat::default(); + imgproc::morphology_ex( + &edges, + &mut closed, + imgproc::MORPH_CLOSE, + &kernel, + core::Point::new(-1, -1), + 2, + core::BORDER_CONSTANT, + Scalar::all(0.0), + ) + .ok()?; + + let mut contours: Vector> = Vector::new(); + imgproc::find_contours( + &closed, + &mut contours, + imgproc::RETR_EXTERNAL, + imgproc::CHAIN_APPROX_SIMPLE, + core::Point::new(0, 0), + ) + .ok()?; + + let img_w = src.cols().max(1) as f32; + let img_h = src.rows().max(1) as f32; + let img_area = img_w * img_h; + let mut best_quad: Option<[Point2f; 4]> = None; + let mut best_score = f32::MIN; + + for contour in contours { + let area = imgproc::contour_area(&contour, false).ok()? as f32; + let area_ratio = area / img_area; + if !(0.25..=0.98).contains(&area_ratio) { + continue; + } + let peri = imgproc::arc_length(&contour, true).ok()?; + let mut approx: Vector = Vector::new(); + imgproc::approx_poly_dp(&contour, &mut approx, 0.02 * peri, true).ok()?; + if approx.len() != 4 { + continue; + } + let mut pts: Vec = approx + .iter() + .map(|p| Point2f::new(p.x as f32, p.y as f32)) + .collect(); + if pts.len() != 4 { + continue; + } + pts.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x))); + let (tl, tr) = if pts[0].x <= pts[1].x { + (pts[0], pts[1]) + } else { + (pts[1], pts[0]) + }; + let (bl, br) = if pts[2].x <= pts[3].x { + (pts[2], pts[3]) + } else { + (pts[3], pts[2]) + }; + let top_w = ((tr.x - tl.x).powi(2) + (tr.y - tl.y).powi(2)).sqrt(); + let bot_w = ((br.x - bl.x).powi(2) + (br.y - bl.y).powi(2)).sqrt(); + let left_h = ((bl.x - tl.x).powi(2) + (bl.y - tl.y).powi(2)).sqrt(); + let right_h = ((br.x - tr.x).powi(2) + (br.y - tr.y).powi(2)).sqrt(); + let width = top_w.max(bot_w).max(1.0); + let height = left_h.max(right_h).max(1.0); + let aspect = width / height; + if !(0.55..=1.9).contains(&aspect) { + continue; + } + let score = area_ratio - (aspect - 1.414).abs() * 0.08; + if score > best_score { + best_score = score; + best_quad = Some([tl, tr, br, bl]); + } + } + + let quad = best_quad?; + let dst_w = 2200; + let dst_h = 1550; + let mut src_vec: Vector = Vector::new(); + for p in quad { + src_vec.push(p); + } + let mut dst_vec: Vector = Vector::new(); + dst_vec.push(Point2f::new(0.0, 0.0)); + dst_vec.push(Point2f::new((dst_w - 1) as f32, 0.0)); + dst_vec.push(Point2f::new((dst_w - 1) as f32, (dst_h - 1) as f32)); + dst_vec.push(Point2f::new(0.0, (dst_h - 1) as f32)); + let m = imgproc::get_perspective_transform(&src_vec, &dst_vec, 0).ok()?; + let mut warped = Mat::default(); + imgproc::warp_perspective( + src, + &mut warped, + &m, + Size::new(dst_w, dst_h), + imgproc::INTER_LINEAR, + core::BORDER_CONSTANT, + Scalar::all(255.0), + ) + .ok()?; + Some(warped) + } + + fn quad_warp_variant(src: &Mat) -> Option { + let mut thresh = Mat::default(); + imgproc::adaptive_threshold( + src, + &mut thresh, + 255.0, + imgproc::ADAPTIVE_THRESH_GAUSSIAN_C, + imgproc::THRESH_BINARY_INV, + 31, + 7.0, + ) + .ok()?; + let kernel = imgproc::get_structuring_element( + imgproc::MORPH_RECT, + Size::new(3, 3), + core::Point::new(-1, -1), + ) + .ok()?; + let mut closed = Mat::default(); + imgproc::morphology_ex( + &thresh, + &mut closed, + imgproc::MORPH_CLOSE, + &kernel, + core::Point::new(-1, -1), + 2, + core::BORDER_CONSTANT, + Scalar::all(0.0), + ) + .ok()?; + + let mut contours: Vector> = Vector::new(); + imgproc::find_contours( + &closed, + &mut contours, + imgproc::RETR_EXTERNAL, + imgproc::CHAIN_APPROX_SIMPLE, + core::Point::new(0, 0), + ) + .ok()?; + + let w = src.cols().max(1) as f32; + let h = src.rows().max(1) as f32; + let img_area = w * h; + let mut best_quad: Option<[Point2f; 4]> = None; + let mut best_score = f32::MIN; + + for contour in contours { + let area = imgproc::contour_area(&contour, false).ok()? as f32; + let area_ratio = area / img_area; + if !(0.01..=0.65).contains(&area_ratio) { + continue; + } + let peri = imgproc::arc_length(&contour, true).ok()?; + let mut approx: Vector = Vector::new(); + imgproc::approx_poly_dp(&contour, &mut approx, 0.03 * peri, true).ok()?; + if approx.len() != 4 { + continue; + } + let rect = imgproc::bounding_rect(&approx).ok()?; + if rect.y < (h * 0.04) as i32 { + continue; + } + let aspect = rect.width as f32 / rect.height.max(1) as f32; + if !(1.5..=5.5).contains(&aspect) { + continue; + } + let score = area_ratio * 0.7 + (1.0 - (aspect - (104.0 / 44.0)).abs() / 4.0) * 0.3; + if score <= best_score { + continue; + } + let mut pts: Vec = approx + .iter() + .map(|p| Point2f::new(p.x as f32, p.y as f32)) + .collect(); + if pts.len() != 4 { + continue; + } + pts.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x))); + let (tl, tr) = if pts[0].x <= pts[1].x { + (pts[0], pts[1]) + } else { + (pts[1], pts[0]) + }; + let (bl, br) = if pts[2].x <= pts[3].x { + (pts[2], pts[3]) + } else { + (pts[3], pts[2]) + }; + best_quad = Some([tl, tr, br, bl]); + best_score = score; + } + + let src_pts = best_quad?; + let dst_w = 104 * 8; + let dst_h = 44 * 8; + let mut src_vec: Vector = Vector::new(); + for p in src_pts { + src_vec.push(p); + } + let mut dst_vec: Vector = Vector::new(); + dst_vec.push(Point2f::new(0.0, 0.0)); + dst_vec.push(Point2f::new((dst_w - 1) as f32, 0.0)); + dst_vec.push(Point2f::new((dst_w - 1) as f32, (dst_h - 1) as f32)); + dst_vec.push(Point2f::new(0.0, (dst_h - 1) as f32)); + let m = imgproc::get_perspective_transform(&src_vec, &dst_vec, 0).ok()?; + let mut warped = Mat::default(); + imgproc::warp_perspective( + src, + &mut warped, + &m, + Size::new(dst_w, dst_h), + imgproc::INTER_LINEAR, + core::BORDER_CONSTANT, + Scalar::all(255.0), + ) + .ok()?; + let gray = mat_to_gray(&warped).ok()?; + Some(DynamicImage::ImageLuma8(gray)) + } + + fn gray_to_mat(gray: &GrayImage) -> opencv::Result { + let mat = Mat::from_slice(gray.as_raw())?; + let reshaped = mat.reshape(1, gray.height() as i32)?; + reshaped.try_clone() + } + + fn mat_to_gray(mat: &Mat) -> opencv::Result { + let width = mat.cols().max(0) as u32; + let height = mat.rows().max(0) as u32; + let data = mat.data_bytes()?; + let mut out = GrayImage::new(width, height); + for y in 0..height { + for x in 0..width { + let idx = (y * width + x) as usize; + out.put_pixel(x, y, Luma([data.get(idx).copied().unwrap_or(255)])); + } + } + Ok(out) + } + + fn clahe_variant(src: &Mat) -> opencv::Result { + let mut dst = Mat::default(); + let mut clahe = imgproc::create_clahe(2.0, core::Size::new(8, 8))?; + clahe.apply(src, &mut dst)?; + Ok(dst) + } + + fn adaptive_variant(src: &Mat) -> opencv::Result { + let mut bin = Mat::default(); + imgproc::adaptive_threshold( + src, + &mut bin, + 255.0, + imgproc::ADAPTIVE_THRESH_GAUSSIAN_C, + imgproc::THRESH_BINARY, + 31, + 7.0, + )?; + let kernel = imgproc::get_structuring_element( + imgproc::MORPH_RECT, + core::Size::new(3, 3), + core::Point::new(-1, -1), + )?; + let mut closed = Mat::default(); + imgproc::morphology_ex( + &bin, + &mut closed, + imgproc::MORPH_CLOSE, + &kernel, + core::Point::new(-1, -1), + 1, + core::BORDER_CONSTANT, + core::Scalar::all(255.0), + )?; + Ok(closed) + } +} /// Result type for scanner operations. pub type Result = std::result::Result; @@ -550,6 +1034,20 @@ fn scan_still_with_diagnostics_inner( timings.decode_attempts_micros = elapsed_micros(decode_started); timings.total_micros = elapsed_micros(started); + #[cfg(all(not(target_arch = "wasm32"), feature = "opencv-fallback"))] + if std::env::var_os("GLYPHNET_SCAN_OPENCV").is_some() + && let Some(decoded) = opencv_fallback::try_decode_with_opencv(image) + { + return Ok(StillScanResult { + decoded, + crop: None, + quad: None, + warp_size: None, + attempts, + timings, + }); + } + if std::env::var_os("GLYPHNET_SCAN_DEBUG").is_some() { eprintln!("scan attempts: {attempts:#?}"); } diff --git a/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-001.jpg b/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-001.jpg new file mode 100644 index 0000000..f954bec Binary files /dev/null and b/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-001.jpg differ diff --git a/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-002.jpg b/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-002.jpg new file mode 100644 index 0000000..7c363d8 Binary files /dev/null and b/crates/glyphnet-testkit/fixtures/corpus/real/app/print-clean-002.jpg differ