fix(sync): correctness overhaul of QRZ + LoTW sync flows

migrations/sync_qrz_lotw_fixes.sql

@@ -0,0 +1,77 @@
+-- Migration: QRZ + LoTW sync correctness fixes
+-- Adds fields required to:
+--   1. Sign LoTW .tq8 files in pure Node (lotw_credentials.p12_password)
+--   2. Track CRL status of stored certificates
+--   3. Drive incremental QRZ/LoTW downloads via last-confirmed timestamps
+--   4. Carry the cross-service 'M' (modified) and 'I' (ignore) flags
+--
+-- Idempotent — safe to re-run.
+--
+-- Prerequisite: postgres-lotw-migration.sql must have been run first to create
+-- the lotw_credentials / lotw_upload_logs / lotw_download_logs tables.
+
+-- 0. contacts: ensure QRZ tracking columns exist (some envs were initialized
+--    before the in-app /install route added these). sync_qrz_lotw_fixes is the
+--    canonical setup for the QRZ flow now.
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_sent      VARCHAR(10);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_rcvd      VARCHAR(10);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_sent_date DATE;
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_rcvd_date DATE;
+CREATE INDEX IF NOT EXISTS idx_contacts_qrz_qsl_sent ON contacts(qrz_qsl_sent);
+CREATE INDEX IF NOT EXISTS idx_contacts_qrz_qsl_rcvd ON contacts(qrz_qsl_rcvd);
+
+-- 1. lotw_credentials: store the encrypted P12 password and CRL state.
+ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS p12_password   TEXT;
+ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS cert_serial    TEXT;
+ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS crl_status     VARCHAR(16);
+ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS crl_checked_at TIMESTAMP;
+CREATE INDEX IF NOT EXISTS idx_lotw_credentials_cert_serial ON lotw_credentials(cert_serial);
+
+-- 2. stations: incremental download bookmarks.
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS lotw_last_qsl_rcvd_date DATE;
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS qrz_last_qsl_rcvd_date  DATE;
+
+-- 3. contacts: prop_mode/sat_name/band_rx/freq_rx columns required to build a
+--    valid LoTW upload + match satellite confirmations correctly. The mode +
+--    band columns already exist; these are the missing TQSL inputs.
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS prop_mode VARCHAR(16);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS sat_name  VARCHAR(32);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS band_rx   VARCHAR(20);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS freq_rx   DECIMAL(10, 6);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS iota      VARCHAR(16);
+ALTER TABLE contacts ADD COLUMN IF NOT EXISTS lotw_qslrdate DATE;
+CREATE INDEX IF NOT EXISTS idx_contacts_prop_mode ON contacts(prop_mode);
+CREATE INDEX IF NOT EXISTS idx_contacts_sat_name  ON contacts(sat_name);
+
+-- 4. contacts: enforce QRZ/LoTW status enum values (Y/N/R/M/I/Q + NULL).
+--    'M' = modified, queued for re-upload after a cross-service confirmation.
+--    'I' = ignored, prop_mode is unsupported by LoTW (INTERNET, RPT).
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.table_constraints
+        WHERE constraint_name = 'contacts_qrz_qsl_sent_check'
+    ) THEN
+        ALTER TABLE contacts
+        ADD CONSTRAINT contacts_qrz_qsl_sent_check
+        CHECK (qrz_qsl_sent IS NULL OR qrz_qsl_sent IN ('Y','N','R','M','I','Q'));
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.table_constraints
+        WHERE constraint_name = 'contacts_lotw_qsl_sent_check'
+    ) THEN
+        ALTER TABLE contacts
+        ADD CONSTRAINT contacts_lotw_qsl_sent_check
+        CHECK (lotw_qsl_sent IS NULL OR lotw_qsl_sent IN ('Y','N','R','M','I','Q'));
+    END IF;
+END$$;
+
+-- 5. stations: ensure DXCC entity / location fields exist (used by .tq8 builder).
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS dxcc_entity_code INTEGER;
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS state_province  VARCHAR(64);
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS county          VARCHAR(64);
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS itu_zone        INTEGER;
+ALTER TABLE stations ADD COLUMN IF NOT EXISTS cq_zone         INTEGER;
+
+SELECT 'sync_qrz_lotw_fixes migration completed' AS message;

package.json

@@ -40,6 +40,7 @@
     "lucide-react": "^0.539.0",
     "next": "^16.2.6",
     "next-auth": "^4.24.11",
+    "node-forge": "^1.4.0",
     "node-html-parser": "^7.0.1",
     "pg": "^8.11.3",
     "react": "^19.1.1",
@@ -56,6 +57,7 @@
     "@types/bcryptjs": "^2.4.6",
     "@types/jsonwebtoken": "^9.0.10",
     "@types/node": "^20",
+    "@types/node-forge": "^1.3.14",
     "@types/pg": "^8.11.2",
     "@types/react": "^19",
     "@types/react-dom": "^19",

scripts/run-migration.mjs

@@ -0,0 +1,59 @@
+// One-shot migration runner.
+// Usage: DATABASE_URL=postgres://... node scripts/run-migration.mjs <sql-file>
+//
+// Wraps the file contents in a transaction so a partial-fail leaves the DB
+// untouched. Idempotent migrations with IF NOT EXISTS / DO $$ BEGIN ... END$$
+// guards are safe to re-run.
+
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import pg from 'pg';
+
+const sqlPath = process.argv[2];
+const dbUrl = process.env.DATABASE_URL;
+
+if (!sqlPath) {
+  console.error('Usage: DATABASE_URL=... node scripts/run-migration.mjs <sql-file>');
+  process.exit(2);
+}
+if (!dbUrl) {
+  console.error('DATABASE_URL is required');
+  process.exit(2);
+}
+
+let sql = readFileSync(resolve(sqlPath), 'utf8');
+
+// Postgres 15/17 don't support `CREATE TRIGGER IF NOT EXISTS` (PG 18+ feature).
+// Rewrite each occurrence as a DROP IF EXISTS + CREATE pair so the SQL works
+// across versions without editing the source migration files.
+sql = sql.replace(
+  /CREATE TRIGGER IF NOT EXISTS\s+(\w+)\s+([\s\S]*?);/gi,
+  (_match, triggerName, body) => {
+    // Extract the table name from the trigger body (e.g., "BEFORE UPDATE ON foo").
+    const tableMatch = body.match(/\bON\s+(\w+)/i);
+    const table = tableMatch ? tableMatch[1] : '';
+    return `DROP TRIGGER IF EXISTS ${triggerName} ON ${table};\nCREATE TRIGGER ${triggerName} ${body};`;
+  }
+);
+
+// Azure Postgres requires SSL; localhost typically doesn't. Detect from URL.
+const ssl = /azure\.com|sslmode=require/i.test(dbUrl) ? { rejectUnauthorized: false } : false;
+const client = new pg.Client({ connectionString: dbUrl, ssl });
+
+const target = dbUrl.replace(/:[^@:/]*@/, ':****@');
+console.log(`Running ${sqlPath} against ${target}`);
+
+try {
+  await client.connect();
+  await client.query('BEGIN');
+  await client.query(sql);
+  await client.query('COMMIT');
+  console.log('Migration applied successfully.');
+} catch (err) {
+  try { await client.query('ROLLBACK'); } catch {}
+  console.error('Migration failed; rolled back.');
+  console.error(err.message);
+  process.exit(1);
+} finally {
+  await client.end();
+}

src/app/api/contacts/qrz-download/route.ts

@@ -3,7 +3,8 @@ import jwt from 'jsonwebtoken';
 import { User } from '@/models/User';
 import { Contact } from '@/models/Contact';
 import { Station } from '@/models/Station';
-import { downloadQSOsFromQRZ } from '@/lib/qrz';
+import { downloadQSOsFromQRZ, matchQRZConfirmation } from '@/lib/qrz';
+import { query } from '@/lib/db';
 
 export async function POST(request: NextRequest) {
   try {
@@ -67,21 +68,44 @@ export async function POST(request: NextRequest) {
         console.log(`Found ${stationContacts.length} unconfirmed contacts for station ${station.callsign}`);
 
         let confirmationsFound = 0;
-
-        // Match QRZ QSOs with our contacts to find confirmations
-        for (const contact of stationContacts) {
+
+        // Annotate each contact with the station callsign so the matcher can
+        // cross-check against QRZ's STATION_CALLSIGN field. ContactData has
+        // station_id but not station_callsign — fill it in from the iterated station.
+        const stationCall = station.callsign;
+        const annotated = stationContacts.map(c => ({ ...c, station_callsign: stationCall }));
+
+        // Match QRZ QSOs with our contacts. Tighter rules: callsign + band +
+        // mode + station_callsign + ±15min, so two QSOs on different bands at
+        // the same minute don't false-match.
+        for (const contact of annotated) {
           for (const qrzQSO of downloadResult.qsos) {
-            if (Contact.matchQSO(contact, qrzQSO)) {
-              console.log(`Found confirmation match for ${contact.callsign} on ${contact.datetime}`);
-
-              // Check if QRZ shows this as confirmed
-              if (qrzQSO.qsl_rcvd === 'Y' || qrzQSO.qsl_sent === 'Y') {
-                console.log(`Marking ${contact.callsign} as QRZ confirmed`);
-                await Contact.updateQrzQsl(contact.id, undefined, 'Y'); // Mark received
-                confirmationsFound++;
-              }
-              break; // Found match, no need to check other QRZ QSOs for this contact
+            if (!matchQRZConfirmation(contact, qrzQSO)) continue;
+
+            // QRZ marks confirmed records with app_qrzlog_status='C'. The legacy
+            // qsl_rcvd / qsl_sent fields aren't always populated, so prefer the
+            // app field when present.
+            const isConfirmed =
+              qrzQSO.app_qrzlog_status?.toUpperCase() === 'C' ||
+              qrzQSO.qsl_rcvd === 'Y' ||
+              qrzQSO.qsl_sent === 'Y';
+            if (!isConfirmed) {
+              break;
+            }
+
+            console.log(`Marking ${contact.callsign} as QRZ confirmed (status=${qrzQSO.app_qrzlog_status ?? '<missing>'})`);
+            await Contact.updateQrzQsl(contact.id, undefined, 'Y');
+            confirmationsFound++;
+
+            // Cross-sync: if LoTW already shipped this QSO, flag for re-upload
+            // so the new qrz_qsl_rcvd value propagates back into LoTW (wavelog 'M').
+            if (contact.lotw_qsl_sent === 'Y') {
+              await query(
+                `UPDATE contacts SET lotw_qsl_sent = 'M', updated_at = NOW() WHERE id = $1`,
+                [contact.id]
+              );
             }
+            break;
           }
         }
 

src/app/api/lotw/certificate/route.ts

@@ -3,6 +3,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { verifyToken } from '@/lib/auth';
 import { query } from '@/lib/db';
+import { encryptString, readCertMetadata } from '@/lib/lotw';
 import { LotwCertificateResponse } from '@/types/lotw';
 
 export async function POST(request: NextRequest) {
@@ -17,6 +18,8 @@ export async function POST(request: NextRequest) {
     const stationId = formData.get('station_id') as string;
     const callsign = formData.get('callsign') as string;
     const certName = formData.get('cert_name') as string;
+    // Optional — TQSL exports without a password are common; node-forge accepts ''.
+    const p12Password = (formData.get('p12_password') as string | null) ?? '';
 
     if (!file || !stationId || !callsign || !certName) {
       return NextResponse.json({
@@ -68,6 +71,22 @@ export async function POST(request: NextRequest) {
       }, { status: 400 });
     }
 
+    // Validate the P12 by parsing it with the supplied password. This catches
+    // wrong-password / corrupt-file uploads before they sit unusable in the DB.
+    let certMeta: { serial: string; notAfter?: Date; dxcc?: number };
+    try {
+      certMeta = readCertMetadata(fileBuffer, p12Password);
+    } catch (parseError) {
+      const msg = parseError instanceof Error ? parseError.message : 'Unknown error';
+      // node-forge throws "PKCS#12 MAC could not be verified" / similar on bad password
+      const isPasswordError = /mac|password|invalid|decrypt/i.test(msg);
+      return NextResponse.json({
+        error: isPasswordError
+          ? 'Could not parse certificate with the supplied password. Re-export from TQSL and re-enter the password.'
+          : `Certificate parse failed: ${msg}`
+      }, { status: 400 });
+    }
+
     // Check if certificate already exists for this station
     const existingCertResult = await query(
       'SELECT id FROM lotw_credentials WHERE station_id = $1 AND is_active = true',
@@ -82,24 +101,37 @@ export async function POST(request: NextRequest) {
       );
     }
 
-    // Store new certificate in lotw_credentials table
+    // Encrypt the P12 password at rest. Empty string is encrypted as well so
+    // the upload route can simply decrypt-or-default; storing NULL would
+    // require a branch in every read path.
+    const encryptedPassword = encryptString(p12Password);
+
+    // Store new certificate + metadata extracted from the P12.
     const insertResult = await query(
       `INSERT INTO lotw_credentials
-       (station_id, name, callsign, p12_cert, cert_created_at, is_active)
-       VALUES ($1, $2, $3, $4, NOW(), true)
-       RETURNING id, cert_created_at`,
-      [parseInt(stationId), certName.trim(), callsign.toUpperCase(), fileBuffer]
+       (station_id, name, callsign, p12_cert, p12_password,
+        cert_serial, cert_created_at, cert_expires_at, is_active)
+       VALUES ($1, $2, $3, $4, $5, $6, NOW(), $7, true)
+       RETURNING id, cert_created_at, cert_expires_at`,
+      [
+        parseInt(stationId),
+        certName.trim(),
+        callsign.toUpperCase(),
+        fileBuffer,
+        encryptedPassword,
+        certMeta.serial,
+        certMeta.notAfter ?? null,
+      ]
     );
 
     const newCredential = insertResult.rows[0];
 
-    // TODO: Extract certificate expiration date from P12 file
-    // This would require additional crypto libraries to parse the certificate
-
     const response: LotwCertificateResponse = {
       success: true,
       credential_id: newCredential.id,
-      // cert_expires_at: expirationDate?.toISOString()
+      cert_expires_at: newCredential.cert_expires_at
+        ? new Date(newCredential.cert_expires_at).toISOString()
+        : undefined,
     };
 
     return NextResponse.json(response);

src/app/api/lotw/download-contact/route.ts

@@ -123,7 +123,11 @@ export async function POST(request: NextRequest) {
     const dateToStr = dateTo.toISOString().split('T')[0];
 
     // Build LoTW download URL with date range
-    const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, dateFromStr, dateToStr);
+    const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, {
+      dateFrom: dateFromStr,
+      dateTo: dateToStr,
+      ownCallsign: contact.station_callsign,
+    });
 
     // Download confirmations from LoTW
     let adifContent: string;