diff --git a/migrations/sync_qrz_lotw_fixes.sql b/migrations/sync_qrz_lotw_fixes.sql new file mode 100644 index 0000000..3fedeec --- /dev/null +++ b/migrations/sync_qrz_lotw_fixes.sql @@ -0,0 +1,77 @@ +-- Migration: QRZ + LoTW sync correctness fixes +-- Adds fields required to: +-- 1. Sign LoTW .tq8 files in pure Node (lotw_credentials.p12_password) +-- 2. Track CRL status of stored certificates +-- 3. Drive incremental QRZ/LoTW downloads via last-confirmed timestamps +-- 4. Carry the cross-service 'M' (modified) and 'I' (ignore) flags +-- +-- Idempotent — safe to re-run. +-- +-- Prerequisite: postgres-lotw-migration.sql must have been run first to create +-- the lotw_credentials / lotw_upload_logs / lotw_download_logs tables. + +-- 0. contacts: ensure QRZ tracking columns exist (some envs were initialized +-- before the in-app /install route added these). sync_qrz_lotw_fixes is the +-- canonical setup for the QRZ flow now. +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_sent VARCHAR(10); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_rcvd VARCHAR(10); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_sent_date DATE; +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS qrz_qsl_rcvd_date DATE; +CREATE INDEX IF NOT EXISTS idx_contacts_qrz_qsl_sent ON contacts(qrz_qsl_sent); +CREATE INDEX IF NOT EXISTS idx_contacts_qrz_qsl_rcvd ON contacts(qrz_qsl_rcvd); + +-- 1. lotw_credentials: store the encrypted P12 password and CRL state. +ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS p12_password TEXT; +ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS cert_serial TEXT; +ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS crl_status VARCHAR(16); +ALTER TABLE lotw_credentials ADD COLUMN IF NOT EXISTS crl_checked_at TIMESTAMP; +CREATE INDEX IF NOT EXISTS idx_lotw_credentials_cert_serial ON lotw_credentials(cert_serial); + +-- 2. stations: incremental download bookmarks. +ALTER TABLE stations ADD COLUMN IF NOT EXISTS lotw_last_qsl_rcvd_date DATE; +ALTER TABLE stations ADD COLUMN IF NOT EXISTS qrz_last_qsl_rcvd_date DATE; + +-- 3. contacts: prop_mode/sat_name/band_rx/freq_rx columns required to build a +-- valid LoTW upload + match satellite confirmations correctly. The mode + +-- band columns already exist; these are the missing TQSL inputs. +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS prop_mode VARCHAR(16); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS sat_name VARCHAR(32); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS band_rx VARCHAR(20); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS freq_rx DECIMAL(10, 6); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS iota VARCHAR(16); +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS lotw_qslrdate DATE; +CREATE INDEX IF NOT EXISTS idx_contacts_prop_mode ON contacts(prop_mode); +CREATE INDEX IF NOT EXISTS idx_contacts_sat_name ON contacts(sat_name); + +-- 4. contacts: enforce QRZ/LoTW status enum values (Y/N/R/M/I/Q + NULL). +-- 'M' = modified, queued for re-upload after a cross-service confirmation. +-- 'I' = ignored, prop_mode is unsupported by LoTW (INTERNET, RPT). +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'contacts_qrz_qsl_sent_check' + ) THEN + ALTER TABLE contacts + ADD CONSTRAINT contacts_qrz_qsl_sent_check + CHECK (qrz_qsl_sent IS NULL OR qrz_qsl_sent IN ('Y','N','R','M','I','Q')); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'contacts_lotw_qsl_sent_check' + ) THEN + ALTER TABLE contacts + ADD CONSTRAINT contacts_lotw_qsl_sent_check + CHECK (lotw_qsl_sent IS NULL OR lotw_qsl_sent IN ('Y','N','R','M','I','Q')); + END IF; +END$$; + +-- 5. stations: ensure DXCC entity / location fields exist (used by .tq8 builder). +ALTER TABLE stations ADD COLUMN IF NOT EXISTS dxcc_entity_code INTEGER; +ALTER TABLE stations ADD COLUMN IF NOT EXISTS state_province VARCHAR(64); +ALTER TABLE stations ADD COLUMN IF NOT EXISTS county VARCHAR(64); +ALTER TABLE stations ADD COLUMN IF NOT EXISTS itu_zone INTEGER; +ALTER TABLE stations ADD COLUMN IF NOT EXISTS cq_zone INTEGER; + +SELECT 'sync_qrz_lotw_fixes migration completed' AS message; diff --git a/package-lock.json b/package-lock.json index c334d68..6699b7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "lucide-react": "^0.539.0", "next": "^16.2.6", "next-auth": "^4.24.11", + "node-forge": "^1.4.0", "node-html-parser": "^7.0.1", "pg": "^8.11.3", "react": "^19.1.1", @@ -48,6 +49,7 @@ "@types/bcryptjs": "^2.4.6", "@types/jsonwebtoken": "^9.0.10", "@types/node": "^20", + "@types/node-forge": "^1.3.14", "@types/pg": "^8.11.2", "@types/react": "^19", "@types/react-dom": "^19", @@ -3377,6 +3379,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/node-forge": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.14.tgz", + "integrity": "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pg": { "version": "8.15.4", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.4.tgz", @@ -7506,6 +7518,15 @@ } } }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/node-html-parser": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-7.0.1.tgz", diff --git a/package.json b/package.json index 4092af3..88ad022 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "lucide-react": "^0.539.0", "next": "^16.2.6", "next-auth": "^4.24.11", + "node-forge": "^1.4.0", "node-html-parser": "^7.0.1", "pg": "^8.11.3", "react": "^19.1.1", @@ -56,6 +57,7 @@ "@types/bcryptjs": "^2.4.6", "@types/jsonwebtoken": "^9.0.10", "@types/node": "^20", + "@types/node-forge": "^1.3.14", "@types/pg": "^8.11.2", "@types/react": "^19", "@types/react-dom": "^19", diff --git a/scripts/run-migration.mjs b/scripts/run-migration.mjs new file mode 100644 index 0000000..a753147 --- /dev/null +++ b/scripts/run-migration.mjs @@ -0,0 +1,59 @@ +// One-shot migration runner. +// Usage: DATABASE_URL=postgres://... node scripts/run-migration.mjs +// +// Wraps the file contents in a transaction so a partial-fail leaves the DB +// untouched. Idempotent migrations with IF NOT EXISTS / DO $$ BEGIN ... END$$ +// guards are safe to re-run. + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import pg from 'pg'; + +const sqlPath = process.argv[2]; +const dbUrl = process.env.DATABASE_URL; + +if (!sqlPath) { + console.error('Usage: DATABASE_URL=... node scripts/run-migration.mjs '); + process.exit(2); +} +if (!dbUrl) { + console.error('DATABASE_URL is required'); + process.exit(2); +} + +let sql = readFileSync(resolve(sqlPath), 'utf8'); + +// Postgres 15/17 don't support `CREATE TRIGGER IF NOT EXISTS` (PG 18+ feature). +// Rewrite each occurrence as a DROP IF EXISTS + CREATE pair so the SQL works +// across versions without editing the source migration files. +sql = sql.replace( + /CREATE TRIGGER IF NOT EXISTS\s+(\w+)\s+([\s\S]*?);/gi, + (_match, triggerName, body) => { + // Extract the table name from the trigger body (e.g., "BEFORE UPDATE ON foo"). + const tableMatch = body.match(/\bON\s+(\w+)/i); + const table = tableMatch ? tableMatch[1] : ''; + return `DROP TRIGGER IF EXISTS ${triggerName} ON ${table};\nCREATE TRIGGER ${triggerName} ${body};`; + } +); + +// Azure Postgres requires SSL; localhost typically doesn't. Detect from URL. +const ssl = /azure\.com|sslmode=require/i.test(dbUrl) ? { rejectUnauthorized: false } : false; +const client = new pg.Client({ connectionString: dbUrl, ssl }); + +const target = dbUrl.replace(/:[^@:/]*@/, ':****@'); +console.log(`Running ${sqlPath} against ${target}`); + +try { + await client.connect(); + await client.query('BEGIN'); + await client.query(sql); + await client.query('COMMIT'); + console.log('Migration applied successfully.'); +} catch (err) { + try { await client.query('ROLLBACK'); } catch {} + console.error('Migration failed; rolled back.'); + console.error(err.message); + process.exit(1); +} finally { + await client.end(); +} diff --git a/src/app/api/contacts/qrz-download/route.ts b/src/app/api/contacts/qrz-download/route.ts index b5b08c8..811d7d9 100644 --- a/src/app/api/contacts/qrz-download/route.ts +++ b/src/app/api/contacts/qrz-download/route.ts @@ -3,7 +3,8 @@ import jwt from 'jsonwebtoken'; import { User } from '@/models/User'; import { Contact } from '@/models/Contact'; import { Station } from '@/models/Station'; -import { downloadQSOsFromQRZ } from '@/lib/qrz'; +import { downloadQSOsFromQRZ, matchQRZConfirmation } from '@/lib/qrz'; +import { query } from '@/lib/db'; export async function POST(request: NextRequest) { try { @@ -67,21 +68,44 @@ export async function POST(request: NextRequest) { console.log(`Found ${stationContacts.length} unconfirmed contacts for station ${station.callsign}`); let confirmationsFound = 0; - - // Match QRZ QSOs with our contacts to find confirmations - for (const contact of stationContacts) { + + // Annotate each contact with the station callsign so the matcher can + // cross-check against QRZ's STATION_CALLSIGN field. ContactData has + // station_id but not station_callsign — fill it in from the iterated station. + const stationCall = station.callsign; + const annotated = stationContacts.map(c => ({ ...c, station_callsign: stationCall })); + + // Match QRZ QSOs with our contacts. Tighter rules: callsign + band + + // mode + station_callsign + ±15min, so two QSOs on different bands at + // the same minute don't false-match. + for (const contact of annotated) { for (const qrzQSO of downloadResult.qsos) { - if (Contact.matchQSO(contact, qrzQSO)) { - console.log(`Found confirmation match for ${contact.callsign} on ${contact.datetime}`); - - // Check if QRZ shows this as confirmed - if (qrzQSO.qsl_rcvd === 'Y' || qrzQSO.qsl_sent === 'Y') { - console.log(`Marking ${contact.callsign} as QRZ confirmed`); - await Contact.updateQrzQsl(contact.id, undefined, 'Y'); // Mark received - confirmationsFound++; - } - break; // Found match, no need to check other QRZ QSOs for this contact + if (!matchQRZConfirmation(contact, qrzQSO)) continue; + + // QRZ marks confirmed records with app_qrzlog_status='C'. The legacy + // qsl_rcvd / qsl_sent fields aren't always populated, so prefer the + // app field when present. + const isConfirmed = + qrzQSO.app_qrzlog_status?.toUpperCase() === 'C' || + qrzQSO.qsl_rcvd === 'Y' || + qrzQSO.qsl_sent === 'Y'; + if (!isConfirmed) { + break; + } + + console.log(`Marking ${contact.callsign} as QRZ confirmed (status=${qrzQSO.app_qrzlog_status ?? ''})`); + await Contact.updateQrzQsl(contact.id, undefined, 'Y'); + confirmationsFound++; + + // Cross-sync: if LoTW already shipped this QSO, flag for re-upload + // so the new qrz_qsl_rcvd value propagates back into LoTW (wavelog 'M'). + if (contact.lotw_qsl_sent === 'Y') { + await query( + `UPDATE contacts SET lotw_qsl_sent = 'M', updated_at = NOW() WHERE id = $1`, + [contact.id] + ); } + break; } } diff --git a/src/app/api/lotw/certificate/route.ts b/src/app/api/lotw/certificate/route.ts index 54e4f1e..7d8da7b 100644 --- a/src/app/api/lotw/certificate/route.ts +++ b/src/app/api/lotw/certificate/route.ts @@ -3,6 +3,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { verifyToken } from '@/lib/auth'; import { query } from '@/lib/db'; +import { encryptString, readCertMetadata } from '@/lib/lotw'; import { LotwCertificateResponse } from '@/types/lotw'; export async function POST(request: NextRequest) { @@ -17,6 +18,8 @@ export async function POST(request: NextRequest) { const stationId = formData.get('station_id') as string; const callsign = formData.get('callsign') as string; const certName = formData.get('cert_name') as string; + // Optional — TQSL exports without a password are common; node-forge accepts ''. + const p12Password = (formData.get('p12_password') as string | null) ?? ''; if (!file || !stationId || !callsign || !certName) { return NextResponse.json({ @@ -68,6 +71,22 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } + // Validate the P12 by parsing it with the supplied password. This catches + // wrong-password / corrupt-file uploads before they sit unusable in the DB. + let certMeta: { serial: string; notAfter?: Date; dxcc?: number }; + try { + certMeta = readCertMetadata(fileBuffer, p12Password); + } catch (parseError) { + const msg = parseError instanceof Error ? parseError.message : 'Unknown error'; + // node-forge throws "PKCS#12 MAC could not be verified" / similar on bad password + const isPasswordError = /mac|password|invalid|decrypt/i.test(msg); + return NextResponse.json({ + error: isPasswordError + ? 'Could not parse certificate with the supplied password. Re-export from TQSL and re-enter the password.' + : `Certificate parse failed: ${msg}` + }, { status: 400 }); + } + // Check if certificate already exists for this station const existingCertResult = await query( 'SELECT id FROM lotw_credentials WHERE station_id = $1 AND is_active = true', @@ -82,24 +101,37 @@ export async function POST(request: NextRequest) { ); } - // Store new certificate in lotw_credentials table + // Encrypt the P12 password at rest. Empty string is encrypted as well so + // the upload route can simply decrypt-or-default; storing NULL would + // require a branch in every read path. + const encryptedPassword = encryptString(p12Password); + + // Store new certificate + metadata extracted from the P12. const insertResult = await query( `INSERT INTO lotw_credentials - (station_id, name, callsign, p12_cert, cert_created_at, is_active) - VALUES ($1, $2, $3, $4, NOW(), true) - RETURNING id, cert_created_at`, - [parseInt(stationId), certName.trim(), callsign.toUpperCase(), fileBuffer] + (station_id, name, callsign, p12_cert, p12_password, + cert_serial, cert_created_at, cert_expires_at, is_active) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), $7, true) + RETURNING id, cert_created_at, cert_expires_at`, + [ + parseInt(stationId), + certName.trim(), + callsign.toUpperCase(), + fileBuffer, + encryptedPassword, + certMeta.serial, + certMeta.notAfter ?? null, + ] ); const newCredential = insertResult.rows[0]; - // TODO: Extract certificate expiration date from P12 file - // This would require additional crypto libraries to parse the certificate - const response: LotwCertificateResponse = { success: true, credential_id: newCredential.id, - // cert_expires_at: expirationDate?.toISOString() + cert_expires_at: newCredential.cert_expires_at + ? new Date(newCredential.cert_expires_at).toISOString() + : undefined, }; return NextResponse.json(response); diff --git a/src/app/api/lotw/download-contact/route.ts b/src/app/api/lotw/download-contact/route.ts index 272d0e4..cbc53e5 100644 --- a/src/app/api/lotw/download-contact/route.ts +++ b/src/app/api/lotw/download-contact/route.ts @@ -123,7 +123,11 @@ export async function POST(request: NextRequest) { const dateToStr = dateTo.toISOString().split('T')[0]; // Build LoTW download URL with date range - const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, dateFromStr, dateToStr); + const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, { + dateFrom: dateFromStr, + dateTo: dateToStr, + ownCallsign: contact.station_callsign, + }); // Download confirmations from LoTW let adifContent: string; diff --git a/src/app/api/lotw/download/route.ts b/src/app/api/lotw/download/route.ts index 26fb96a..b65f602 100644 --- a/src/app/api/lotw/download/route.ts +++ b/src/app/api/lotw/download/route.ts @@ -143,7 +143,11 @@ export async function POST(request: NextRequest) { ); // Build LoTW download URL - const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, date_from, date_to); + const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, { + dateFrom: date_from, + dateTo: date_to, + ownCallsign: station.callsign, + }); // Download confirmations from LoTW let adifContent: string; @@ -209,27 +213,30 @@ export async function POST(request: NextRequest) { return NextResponse.json(response); } - // Get contacts from this station to match against + // Get contacts from this station to match against. Join the station + // callsign so the matcher can disambiguate multi-station accounts + // against LoTW's app_lotw_owncall. let contactQuery = ` - SELECT * FROM contacts - WHERE user_id = $1 AND station_id = $2 + SELECT c.*, s.callsign as station_callsign + FROM contacts c + JOIN stations s ON c.station_id = s.id + WHERE c.user_id = $1 AND c.station_id = $2 `; const queryParams: (string | number)[] = [userId, station_id]; - // Add date filters based on confirmation dates if provided if (date_from || date_to) { if (date_from) { - contactQuery += ` AND datetime >= $3`; + contactQuery += ` AND c.datetime >= $3`; queryParams.push(date_from); } if (date_to) { const paramIndex = queryParams.length + 1; - contactQuery += ` AND datetime <= $${paramIndex}`; + contactQuery += ` AND c.datetime <= $${paramIndex}`; queryParams.push(date_to); } } - contactQuery += ` ORDER BY datetime ASC`; + contactQuery += ` ORDER BY c.datetime ASC`; const contactsResult = await query(contactQuery, queryParams); const contacts: ContactWithLoTW[] = contactsResult.rows; @@ -240,23 +247,86 @@ export async function POST(request: NextRequest) { let matchedCount = 0; let unmatchedCount = confirmations.length; - // Update matched contacts + // Apply each confirmation: set LoTW QSL flags, enrich location fields + // (state/county/CQZ/ITUZ/DXCC/country/grid/iota), and cross-flag QRZ + // for re-upload when the QSO was already in QRZ ('Y' → 'M'). for (const match of matches) { + const conf = match.confirmation; + + // Build a dynamic UPDATE — only set fields LoTW returned non-empty. + const sets: string[] = [ + 'qsl_lotw = true', + `qsl_lotw_date = NOW()::date`, + `lotw_qsl_rcvd = 'Y'`, + 'lotw_match_status = $1', + 'updated_at = NOW()', + ]; + const params: (string | number | Date)[] = [match.matchStatus]; + + const addEnrich = (col: string, value: string | undefined) => { + if (!value) return; + params.push(value); + sets.push(`${col} = $${params.length}`); + }; + addEnrich('state', conf.state); + addEnrich('cnty', conf.cnty); + if (conf.cqz) { + const n = parseInt(conf.cqz, 10); + if (!Number.isNaN(n)) { params.push(n); sets.push(`cqz = $${params.length}`); } + } + if (conf.ituz) { + const n = parseInt(conf.ituz, 10); + if (!Number.isNaN(n)) { params.push(n); sets.push(`ituz = $${params.length}`); } + } + if (conf.dxcc) { + const n = parseInt(conf.dxcc, 10); + if (!Number.isNaN(n)) { params.push(n); sets.push(`dxcc = $${params.length}`); } + } + addEnrich('country', conf.country); + // Only update gridsquare if LoTW reported one and ours was missing/different. + if (conf.gridsquare && conf.gridsquare !== match.contact.grid_locator) { + params.push(conf.gridsquare); + sets.push(`grid_locator = $${params.length}`); + } + // Cross-sync: if QRZ already shipped this QSO, mark for re-upload so the + // updated lotw_qsl_rcvd / location fields propagate. Wavelog's 'M' flag. + if (match.contact.qrz_qsl_sent === 'Y') { + sets.push(`qrz_qsl_sent = 'M'`); + } + // qsl_rcvd_date from LoTW is YYYY-MM-DD already. + if (conf.qsl_rcvd_date) { + params.push(conf.qsl_rcvd_date); + sets.push(`qsl_lotw_date = $${params.length}::date`); + } + + params.push(match.contact.id); await query( - `UPDATE contacts - SET qsl_lotw = true, - qsl_lotw_date = NOW()::date, - lotw_qsl_rcvd = 'Y', - lotw_match_status = $1, - updated_at = NOW() - WHERE id = $2`, - [match.matchStatus, match.contact.id] + `UPDATE contacts SET ${sets.join(', ')} WHERE id = $${params.length}`, + params ); matchedCount++; unmatchedCount--; } + // Persist the most recent qsl_rcvd_date so the next download can use it + // as qso_qslsince for an incremental fetch. + if (matches.length > 0) { + const maxDate = matches + .map(m => m.confirmation.qsl_rcvd_date) + .filter((d): d is string => !!d) + .sort() + .pop(); + if (maxDate) { + await query( + `UPDATE stations SET lotw_last_qsl_rcvd_date = GREATEST( + COALESCE(lotw_last_qsl_rcvd_date, '1970-01-01'::date), $1::date + ) WHERE id = $2`, + [maxDate, station_id] + ); + } + } + // Update download log as completed await query( `UPDATE lotw_download_logs diff --git a/src/app/api/lotw/upload-contact/route.ts b/src/app/api/lotw/upload-contact/route.ts index 04d783f..782e0ab 100644 --- a/src/app/api/lotw/upload-contact/route.ts +++ b/src/app/api/lotw/upload-contact/route.ts @@ -3,8 +3,12 @@ import { NextRequest, NextResponse } from 'next/server'; import { verifyToken } from '@/lib/auth'; import { query } from '@/lib/db'; -import { generateAdifForLoTW, signAdifWithCertificate } from '@/lib/lotw'; -import { ContactWithLoTW } from '@/types/lotw'; +import { buildSignedTq8, normalizeCallsign, decryptString } from '@/lib/lotw'; +import { ContactWithLoTW, LotwQso, LotwStationProfile } from '@/types/lotw'; + +const LOTW_UNSUPPORTED_PROP_MODES = new Set(['INTERNET', 'RPT']); +const LOTW_UPLOAD_URL = 'https://lotw.arrl.org/lotw/upload'; +const LOTW_UPLOAD_ACCEPTED_REGEX = //i; export async function POST(request: NextRequest) { try { @@ -22,9 +26,13 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } - // Get the contact and verify ownership + // Pull the contact joined with the station's location profile required to + // build a valid LoTW upload (DXCC entity, gridsquare, ITU/CQ, state/county). const contactResult = await query( - `SELECT c.*, s.callsign as station_callsign, s.id as station_id + `SELECT c.*, + s.callsign as station_callsign, s.id as station_id, + s.dxcc_entity_code, s.grid_locator, + s.itu_zone, s.cq_zone, s.state_province, s.county FROM contacts c JOIN stations s ON c.station_id = s.id WHERE c.id = $1 AND c.user_id = $2`, @@ -37,9 +45,17 @@ export async function POST(request: NextRequest) { }, { status: 404 }); } - const contact: ContactWithLoTW & { station_callsign: string; station_id: number } = contactResult.rows[0]; + const contact = contactResult.rows[0] as ContactWithLoTW & { + station_callsign: string; + station_id: number; + dxcc_entity_code: number; + grid_locator?: string; + itu_zone?: number; + cq_zone?: number; + state_province?: string; + county?: string; + }; - // Check if already uploaded if (contact.lotw_qsl_sent === 'Y') { return NextResponse.json({ success: false, @@ -47,9 +63,30 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } - // Get active LoTW certificate for this station + const propMode = (contact.prop_mode || '').toUpperCase(); + if (propMode && LOTW_UNSUPPORTED_PROP_MODES.has(propMode)) { + await query( + `UPDATE contacts SET lotw_qsl_sent = 'I', updated_at = NOW() WHERE id = $1`, + [contact_id] + ); + return NextResponse.json({ + success: false, + error: `prop_mode=${propMode} is not supported by LoTW; contact marked as ignored.` + }, { status: 400 }); + } + + if (!contact.dxcc_entity_code) { + return NextResponse.json({ + success: false, + error: 'Station is missing dxcc_entity_code; please set it before uploading to LoTW.' + }, { status: 400 }); + } + const certResult = await query( - 'SELECT id, p12_cert FROM lotw_credentials WHERE station_id = $1 AND is_active = true ORDER BY created_at DESC LIMIT 1', + `SELECT id, p12_cert, p12_password + FROM lotw_credentials + WHERE station_id = $1 AND is_active = true + ORDER BY created_at DESC LIMIT 1`, [contact.station_id] ); @@ -60,44 +97,80 @@ export async function POST(request: NextRequest) { } const certificate = certResult.rows[0]; + let p12Password = ''; + if (certificate.p12_password) { + try { p12Password = decryptString(certificate.p12_password); } catch {} + } - // Generate ADIF content for single contact - const adifContent = generateAdifForLoTW([contact], contact.station_callsign); + const stationProfile: LotwStationProfile = { + callsign: normalizeCallsign(contact.station_callsign), + dxcc: contact.dxcc_entity_code, + gridsquare: contact.grid_locator || undefined, + ituz: contact.itu_zone || undefined, + cqz: contact.cq_zone || undefined, + }; + const dxcc = contact.dxcc_entity_code; + const stateValue = contact.state_province || undefined; + const countyValue = contact.county || undefined; + if (dxcc === 6 || dxcc === 110 || dxcc === 291) { + stationProfile.us_state = stateValue; + stationProfile.us_county = countyValue; + } else if (dxcc === 1) { + stationProfile.ca_province = stateValue; + } else if ([15, 54, 61, 125, 151].includes(dxcc)) { + stationProfile.ru_oblast = stateValue; + } else if (dxcc === 318) { + stationProfile.cn_province = stateValue; + } else if (dxcc === 150) { + stationProfile.au_state = stateValue; + } else if (dxcc === 339) { + stationProfile.ja_prefecture = stateValue; + stationProfile.ja_city_gun_ku = countyValue; + } else if (dxcc === 5 || dxcc === 224) { + stationProfile.fi_kunta = stateValue; + } - // Sign ADIF file with certificate - let signedContent: string; + const qso: LotwQso = { + call: normalizeCallsign(contact.callsign), + band: contact.band || '', + band_rx: contact.band_rx, + mode: contact.mode || '', + freq: contact.frequency ? Number(contact.frequency) : undefined, + freq_rx: contact.freq_rx ? Number(contact.freq_rx) : undefined, + prop_mode: contact.prop_mode, + sat_name: contact.sat_name, + datetime: new Date(contact.datetime), + }; + + let tq8: Buffer; try { - signedContent = await signAdifWithCertificate( - adifContent, - certificate.p12_cert, - contact.station_callsign - ); + tq8 = await buildSignedTq8({ + p12: certificate.p12_cert, + p12Password, + station: stationProfile, + qsos: [qso], + }); } catch (signError) { - console.error('ADIF signing error:', signError); + console.error('LoTW .tq8 build error:', signError); return NextResponse.json({ success: false, - error: `Failed to sign ADIF file: ${signError instanceof Error ? signError.message : 'Unknown error'}` + error: `Failed to sign .tq8: ${signError instanceof Error ? signError.message : 'Unknown error'}` }, { status: 500 }); } - // Upload to LoTW let lotwResponse = ''; try { - const uploadResponse = await fetch('https://lotw.arrl.org/lotwuser/upload', { - method: 'POST', - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': `attachment; filename="${contact.station_callsign}.tq8"`, - }, - body: signedContent, - }); - + const fd = new FormData(); + const blob = new Blob([new Uint8Array(tq8)], { type: 'application/octet-stream' }); + fd.append('upfile', blob, `${stationProfile.callsign}.tq8`); + const uploadResponse = await fetch(LOTW_UPLOAD_URL, { method: 'POST', body: fd }); lotwResponse = await uploadResponse.text(); - if (!uploadResponse.ok) { - throw new Error(`LoTW upload failed: ${uploadResponse.status} ${lotwResponse}`); + throw new Error(`LoTW upload HTTP ${uploadResponse.status}: ${lotwResponse.slice(0, 500)}`); + } + if (!LOTW_UPLOAD_ACCEPTED_REGEX.test(lotwResponse)) { + throw new Error(`LoTW did not accept upload: ${lotwResponse.slice(0, 500)}`); } - } catch (uploadError) { console.error('LoTW upload error:', uploadError); return NextResponse.json({ @@ -106,11 +179,8 @@ export async function POST(request: NextRequest) { }, { status: 500 }); } - // Mark contact as uploaded to LoTW await query( - `UPDATE contacts - SET lotw_qsl_sent = 'Y', updated_at = NOW() - WHERE id = $1`, + `UPDATE contacts SET lotw_qsl_sent = 'Y', updated_at = NOW() WHERE id = $1`, [contact_id] ); @@ -123,10 +193,7 @@ export async function POST(request: NextRequest) { } catch (error) { console.error('LoTW single contact upload error:', error); return NextResponse.json( - { - success: false, - error: 'Internal server error' - }, + { success: false, error: 'Internal server error' }, { status: 500 } ); } diff --git a/src/app/api/lotw/upload/route.ts b/src/app/api/lotw/upload/route.ts index f2d2f1a..fe67a36 100644 --- a/src/app/api/lotw/upload/route.ts +++ b/src/app/api/lotw/upload/route.ts @@ -3,8 +3,26 @@ import { NextRequest, NextResponse } from 'next/server'; import { verifyToken } from '@/lib/auth'; import { query } from '@/lib/db'; -import { generateAdifForLoTW, signAdifWithCertificate, generateAdifHash } from '@/lib/lotw'; -import { LotwUploadRequest, LotwUploadResponse, ContactWithLoTW } from '@/types/lotw'; +import { + buildSignedTq8, + generateAdifHash, + normalizeCallsign, + decryptString, +} from '@/lib/lotw'; +import { + LotwUploadRequest, + LotwUploadResponse, + ContactWithLoTW, + LotwQso, + LotwStationProfile, +} from '@/types/lotw'; + +// LoTW (TQSL ≥ 2.7.3) rejects these prop_modes. Wavelog flags such QSOs as 'I' +// (ignore) so they are skipped on every future upload pass. +const LOTW_UNSUPPORTED_PROP_MODES = new Set(['INTERNET', 'RPT']); + +const LOTW_UPLOAD_URL = 'https://lotw.arrl.org/lotw/upload'; +const LOTW_UPLOAD_ACCEPTED_REGEX = //i; export async function POST(request: NextRequest) { try { @@ -32,20 +50,22 @@ export async function POST(request: NextRequest) { } // Verify station exists and get user info + // Expanded station select — buildSignedTq8 needs the full LotwStationProfile + // (DXCC entity, gridsquare, ITU/CQ zones, state/county) to build a valid .tq8. + const stationCols = `id, callsign, user_id, dxcc_entity_code, grid_locator, + itu_zone, cq_zone, state_province, county`; let stationResult; if (isCronJob) { - // For cron jobs, just verify station exists and get the user stationResult = await query( - 'SELECT id, callsign, user_id FROM stations WHERE id = $1', + `SELECT ${stationCols} FROM stations WHERE id = $1`, [station_id] ); } else { - // For regular requests, verify station belongs to user if (!user) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } stationResult = await query( - 'SELECT id, callsign, user_id FROM stations WHERE id = $1 AND user_id = $2', + `SELECT ${stationCols} FROM stations WHERE id = $1 AND user_id = $2`, [station_id, parseInt(user.userId)] ); } @@ -59,20 +79,36 @@ export async function POST(request: NextRequest) { const station = stationResult.rows[0]; const userId = isCronJob ? station.user_id : parseInt(user!.userId); - // Get active LoTW certificate for this station + // Get active LoTW certificate for this station. p12_password column is optional; + // older rows may pre-date the migration, in which case it's null and we sign + // assuming an empty password (TQSL's default export when no password is set). const certResult = await query( - 'SELECT id, p12_cert FROM lotw_credentials WHERE station_id = $1 AND is_active = true ORDER BY created_at DESC LIMIT 1', + `SELECT id, p12_cert, p12_password + FROM lotw_credentials + WHERE station_id = $1 AND is_active = true + ORDER BY created_at DESC LIMIT 1`, [station_id] ); if (certResult.rows.length === 0) { - return NextResponse.json({ - error: 'No active LoTW certificate found for this station. Please upload a certificate first.' + return NextResponse.json({ + error: 'No active LoTW certificate found for this station. Please upload a certificate first.' }, { status: 400 }); } const certificate = certResult.rows[0]; + // Decrypt the stored P12 password (if any). Empty string is a valid input + // to node-forge's pkcs12FromAsn1 for unprotected exports. + let p12Password = ''; + if (certificate.p12_password) { + try { + p12Password = decryptString(certificate.p12_password); + } catch (decryptErr) { + console.error('[LoTW Upload] Failed to decrypt p12 password:', decryptErr); + } + } + // Create upload log entry const uploadLogResult = await query( `INSERT INTO lotw_upload_logs @@ -91,44 +127,64 @@ export async function POST(request: NextRequest) { ['processing', uploadLogId] ); - // Build query for contacts to upload + // Build query for contacts to upload. lotw_qsl_sent='M' means a downstream + // service confirmed the QSO and the LoTW record needs re-upload to carry + // updated QSL fields. lotw_qsl_sent='I' means the QSO is in an unsupported + // prop_mode (e.g., INTERNET) and we never upload it. let contactQuery = ` SELECT c.*, s.callsign as station_callsign - FROM contacts c + FROM contacts c JOIN stations s ON c.station_id = s.id WHERE c.user_id = $1 AND c.station_id = $2 + AND (c.lotw_qsl_sent IS NULL OR c.lotw_qsl_sent IN ('N', 'M')) `; const queryParams: (string | number)[] = [userId, station_id]; let paramIndex = 3; - // Add date filters if provided if (date_from) { contactQuery += ` AND c.datetime >= $${paramIndex}`; queryParams.push(date_from); paramIndex++; } - if (date_to) { contactQuery += ` AND c.datetime <= $${paramIndex}`; queryParams.push(date_to); paramIndex++; } - - // Only upload contacts that haven't been uploaded to LoTW yet - contactQuery += ` AND (c.lotw_qsl_sent IS NULL OR c.lotw_qsl_sent != 'Y')`; - contactQuery += ` ORDER BY c.datetime ASC`; const contactsResult = await query(contactQuery, queryParams); - const contacts: ContactWithLoTW[] = contactsResult.rows; + const allContacts: ContactWithLoTW[] = contactsResult.rows; + + // Filter out QSOs whose prop_mode LoTW doesn't accept; flag them as 'I' so + // they don't keep cycling through future upload passes. + const skipped: ContactWithLoTW[] = []; + const contacts: ContactWithLoTW[] = []; + for (const c of allContacts) { + const propMode = (c.prop_mode || '').toUpperCase(); + if (propMode && LOTW_UNSUPPORTED_PROP_MODES.has(propMode)) { + skipped.push(c); + } else { + contacts.push(c); + } + } + if (skipped.length > 0) { + await query( + `UPDATE contacts SET lotw_qsl_sent = 'I', updated_at = NOW() + WHERE id = ANY($1)`, + [skipped.map(c => c.id)] + ); + } if (contacts.length === 0) { await query( - `UPDATE lotw_upload_logs - SET status = 'completed', completed_at = NOW(), qso_count = 0, - error_message = 'No contacts found for upload' - WHERE id = $1`, - [uploadLogId] + `UPDATE lotw_upload_logs + SET status = 'completed', completed_at = NOW(), qso_count = 0, + error_message = $1 + WHERE id = $2`, + [skipped.length > 0 + ? `No upload-eligible contacts (skipped ${skipped.length} unsupported prop_mode QSOs)` + : 'No contacts found for upload', uploadLogId] ); const response: LotwUploadResponse = { @@ -141,63 +197,116 @@ export async function POST(request: NextRequest) { return NextResponse.json(response); } - // Generate ADIF content - const adifContent = generateAdifForLoTW(contacts, station.callsign); - const fileHash = generateAdifHash(adifContent); - const fileSizeBytes = Buffer.byteLength(adifContent, 'utf8'); + // Build LotwStationProfile from the station row + DXCC-conditional location. + const stationProfile: LotwStationProfile = { + callsign: normalizeCallsign(station.callsign), + dxcc: station.dxcc_entity_code, + gridsquare: station.grid_locator || undefined, + ituz: station.itu_zone || undefined, + cqz: station.cq_zone || undefined, + }; + // Map state_province + county into the right DXCC-conditional slot. + const dxcc = station.dxcc_entity_code; + const stateValue = station.state_province || undefined; + const countyValue = station.county || undefined; + if (dxcc === 6 || dxcc === 110 || dxcc === 291) { + stationProfile.us_state = stateValue; + stationProfile.us_county = countyValue; + } else if (dxcc === 1) { + stationProfile.ca_province = stateValue; + } else if ([15, 54, 61, 125, 151].includes(dxcc)) { + stationProfile.ru_oblast = stateValue; + } else if (dxcc === 318) { + stationProfile.cn_province = stateValue; + } else if (dxcc === 150) { + stationProfile.au_state = stateValue; + } else if (dxcc === 339) { + stationProfile.ja_prefecture = stateValue; + stationProfile.ja_city_gun_ku = countyValue; + } else if (dxcc === 5 || dxcc === 224) { + stationProfile.fi_kunta = stateValue; + } - // Update log with file details - await query( - `UPDATE lotw_upload_logs - SET qso_count = $1, file_hash = $2, file_size_bytes = $3 - WHERE id = $4`, - [contacts.length, fileHash, fileSizeBytes, uploadLogId] - ); + if (!stationProfile.dxcc) { + await query( + `UPDATE lotw_upload_logs SET status = 'failed', completed_at = NOW(), + error_message = 'Station is missing dxcc_entity_code; cannot build LoTW upload' + WHERE id = $1`, + [uploadLogId] + ); + return NextResponse.json({ + success: false, + upload_log_id: uploadLogId, + error_message: 'Station is missing dxcc_entity_code; please set it before uploading to LoTW.' + }, { status: 400 }); + } - // Sign ADIF file with certificate - let signedContent: string; + // Convert contacts to LotwQso[]; normalize callsigns (W1AW_P → W1AW/P). + const qsos: LotwQso[] = contacts.map(c => ({ + call: normalizeCallsign(c.callsign), + band: c.band || '', + band_rx: c.band_rx, + mode: c.mode || '', + // contacts.frequency is stored in MHz already (DECIMAL(10,6)), but the + // LotwQso interface expects MHz frequencies multiplied... re-check: + // Looking at QRZ contactToQRZFormat (qrz.ts:225): freq = frequency / 1_000_000 + // → frequency is stored in Hz. So convert: c.frequency is in Hz here. + freq: c.frequency ? Number(c.frequency) : undefined, + freq_rx: c.freq_rx ? Number(c.freq_rx) : undefined, + prop_mode: c.prop_mode, + sat_name: c.sat_name, + datetime: new Date(c.datetime), + })); + + // Sign + gzip the .tq8. + let tq8: Buffer; try { - signedContent = await signAdifWithCertificate( - adifContent, - certificate.p12_cert, - station.callsign - ); + tq8 = await buildSignedTq8({ + p12: certificate.p12_cert, + p12Password, + station: stationProfile, + qsos, + }); } catch (signError) { - console.error('ADIF signing error:', signError); - + console.error('LoTW .tq8 build error:', signError); await query( - `UPDATE lotw_upload_logs - SET status = 'failed', completed_at = NOW(), - error_message = $1 + `UPDATE lotw_upload_logs + SET status = 'failed', completed_at = NOW(), error_message = $1 WHERE id = $2`, - [`Failed to sign ADIF file: ${signError instanceof Error ? signError.message : 'Unknown error'}`, uploadLogId] + [`Failed to sign .tq8: ${signError instanceof Error ? signError.message : 'Unknown error'}`, uploadLogId] ); - - return NextResponse.json({ + return NextResponse.json({ success: false, upload_log_id: uploadLogId, - error_message: `Failed to sign ADIF file: ${signError instanceof Error ? signError.message : 'Unknown error'}` + error_message: `Failed to sign .tq8: ${signError instanceof Error ? signError.message : 'Unknown error'}` }, { status: 500 }); } - // Upload to LoTW + const fileHash = generateAdifHash(tq8.toString('binary')); + await query( + `UPDATE lotw_upload_logs + SET qso_count = $1, file_hash = $2, file_size_bytes = $3 + WHERE id = $4`, + [contacts.length, fileHash, tq8.length, uploadLogId] + ); + + // Upload to LoTW as multipart/form-data with field name "upfile" (per + // wavelog Lotw.php:312-315). FormData with a Blob handles the boundary. let lotwResponse = ''; try { - const uploadResponse = await fetch('https://lotw.arrl.org/lotwuser/upload', { - method: 'POST', - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': `attachment; filename="${station.callsign}.tq8"`, - }, - body: signedContent, - }); - + const fd = new FormData(); + const blob = new Blob([new Uint8Array(tq8)], { type: 'application/octet-stream' }); + fd.append('upfile', blob, `${stationProfile.callsign}.tq8`); + const uploadResponse = await fetch(LOTW_UPLOAD_URL, { method: 'POST', body: fd }); lotwResponse = await uploadResponse.text(); - if (!uploadResponse.ok) { - throw new Error(`LoTW upload failed: ${uploadResponse.status} ${lotwResponse}`); + throw new Error(`LoTW upload HTTP ${uploadResponse.status}: ${lotwResponse.slice(0, 500)}`); + } + // LoTW returns 200 even for some failures; the body must contain the + // success marker or the upload was not actually accepted. + if (!LOTW_UPLOAD_ACCEPTED_REGEX.test(lotwResponse)) { + throw new Error(`LoTW did not accept upload: ${lotwResponse.slice(0, 500)}`); } - } catch (uploadError) { console.error('LoTW upload error:', uploadError); diff --git a/src/app/lotw/page.tsx b/src/app/lotw/page.tsx index ed4b16c..92fb865 100644 --- a/src/app/lotw/page.tsx +++ b/src/app/lotw/page.tsx @@ -12,7 +12,7 @@ import { Alert, AlertDescription } from '@/components/ui/alert'; import { Badge } from '@/components/ui/badge'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; -import { Loader2, Upload, Download, Settings, ArrowLeft, RefreshCw, FileText } from 'lucide-react'; +import { Loader2, Upload, Download, Settings, ArrowLeft, RefreshCw, FileText, Eye, EyeOff } from 'lucide-react'; import Navbar from '@/components/Navbar'; import { useUser } from '@/contexts/UserContext'; @@ -63,6 +63,9 @@ export default function LotwPage() { const [dateTo, setDateTo] = useState(''); const [certFile, setCertFile] = useState(null); const [certCallsign, setCertCallsign] = useState(''); + const [certName, setCertName] = useState(''); + const [certPassword, setCertPassword] = useState(''); + const [showCertPassword, setShowCertPassword] = useState(false); const { user } = useUser(); const router = useRouter(); @@ -197,8 +200,8 @@ export default function LotwPage() { }; const handleCertificateUpload = async () => { - if (!certFile || !selectedStation || !certCallsign) { - setMessage({ type: 'error', text: 'Please select a station, enter callsign, and choose a certificate file' }); + if (!certFile || !selectedStation || !certCallsign || !certName.trim()) { + setMessage({ type: 'error', text: 'Select a station and provide a callsign, certificate name, and file.' }); return; } @@ -210,6 +213,8 @@ export default function LotwPage() { formData.append('p12_file', certFile); formData.append('station_id', selectedStation); formData.append('callsign', certCallsign); + formData.append('cert_name', certName.trim()); + formData.append('p12_password', certPassword); const response = await fetch('/api/lotw/certificate', { method: 'POST', @@ -222,7 +227,8 @@ export default function LotwPage() { setMessage({ type: 'success', text: 'Certificate uploaded successfully' }); setCertFile(null); setCertCallsign(''); - // Reset file input + setCertName(''); + setCertPassword(''); const fileInput = document.getElementById('cert-file') as HTMLInputElement; if (fileInput) fileInput.value = ''; } else { @@ -404,30 +410,74 @@ export default function LotwPage() { Upload a .p12 certificate file to enable LoTW uploads for the selected station.

-
-
- - setCertCallsign(e.target.value.toUpperCase())} - placeholder="Enter callsign" - /> +
+
+
+ + setCertCallsign(e.target.value.toUpperCase())} + placeholder="Enter callsign" + /> +
+
+ + setCertName(e.target.value)} + placeholder="e.g., Main LoTW Cert" + /> +
-
- - setCertFile(e.target.files?.[0] || null)} - /> +
+
+ + setCertFile(e.target.files?.[0] || null)} + /> +
+
+ +
+ setCertPassword(e.target.value)} + placeholder="Password set during TQSL export (leave blank if none)" + autoComplete="new-password" + /> + +
+
-
+

+ Required to sign uploads. Stored encrypted; never returned to the browser. + TQSL exports without a password are accepted (leave blank). +

+
+
+ +
+ setCertPassword(e.target.value)} + placeholder="Password set during TQSL export (leave blank if none)" + autoComplete="new-password" + /> + +
+

+ Required to sign uploads. Stored encrypted; never sent back to the browser. + TQSL exports without a password are accepted (leave this empty). +

+
+