feat(webhooks): add signed webhook delivery with retries

app/backend/prisma/migrations/20260626150818_add_webhook_delivery/migration.sql

@@ -0,0 +1,79 @@
+-- CreateTable
+CREATE TABLE "SorobanTransaction" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "claimId" TEXT,
+    "operation" TEXT NOT NULL,
+    "packageId" TEXT,
+    "txHash" TEXT,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "attemptCount" INTEGER NOT NULL DEFAULT 0,
+    "maxAttempts" INTEGER NOT NULL DEFAULT 5,
+    "lastRetryAt" DATETIME,
+    "nextRetryAt" DATETIME,
+    "lastError" TEXT,
+    "errorType" TEXT,
+    "isRetryable" BOOLEAN NOT NULL DEFAULT true,
+    "operatorAddress" TEXT,
+    "recipientAddress" TEXT,
+    "amount" TEXT,
+    "tokenAddress" TEXT,
+    "initiatedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "submittedAt" DATETIME,
+    "confirmedAt" DATETIME,
+    "failedAt" DATETIME,
+    "expiredAt" DATETIME,
+    "correlationId" TEXT,
+    "metadata" JSONB,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "SorobanTransaction_claimId_fkey" FOREIGN KEY ("claimId") REFERENCES "Claim" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "WebhookDelivery" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "url" TEXT NOT NULL,
+    "payload" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "retryCount" INTEGER NOT NULL DEFAULT 0,
+    "lastError" TEXT,
+    "lastAttemptAt" DATETIME,
+    "sentAt" DATETIME,
+    "entityId" TEXT,
+    "entityType" TEXT,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_claimId_idx" ON "SorobanTransaction"("claimId");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_status_idx" ON "SorobanTransaction"("status");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_operation_idx" ON "SorobanTransaction"("operation");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_txHash_idx" ON "SorobanTransaction"("txHash");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_nextRetryAt_idx" ON "SorobanTransaction"("nextRetryAt");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_correlationId_idx" ON "SorobanTransaction"("correlationId");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_attemptCount_idx" ON "SorobanTransaction"("attemptCount");
+
+-- CreateIndex
+CREATE INDEX "SorobanTransaction_isRetryable_nextRetryAt_idx" ON "SorobanTransaction"("isRetryable", "nextRetryAt");
+
+-- CreateIndex
+CREATE INDEX "WebhookDelivery_status_idx" ON "WebhookDelivery"("status");
+
+-- CreateIndex
+CREATE INDEX "WebhookDelivery_createdAt_idx" ON "WebhookDelivery"("createdAt");
+
+-- CreateIndex
+CREATE INDEX "WebhookDelivery_entityId_entityType_idx" ON "WebhookDelivery"("entityId", "entityType");

app/backend/src/aid/aid.module.ts

@@ -1,7 +1,7 @@
 import { Module } from '@nestjs/common';
 import { AidService } from './aid.service';
 import { AidController } from './aid.controller';
-import { RedisService } from 'cache/redis.service';
+import { RedisService } from '../../cache/redis.service';
 import { HmacModule } from '../common/hmac/hmac.module';
 import { WebhookHmacGuard } from '../common/guards/webhook-hmac.guard';
 import { MetricsModule } from '../observability/metrics/metrics.module';

app/backend/src/jobs/jobs.controller.ts

@@ -13,6 +13,7 @@ export class JobsController {
     @InjectQueue('onchain') private onchainQueue: Queue,
     @InjectQueue(RETENTION_PURGE_QUEUE) private retentionPurgeQueue: Queue,
     @InjectQueue('dead-letter') private deadLetterQueue: Queue,
+    @InjectQueue('webhooks') private webhooksQueue: Queue,
   ) {}
 
   @ApiOperation({
@@ -59,6 +60,7 @@ export class JobsController {
       onchain: await this.getQueueStatus(this.onchainQueue),
       'retention-purge': await this.getQueueStatus(this.retentionPurgeQueue),
       'dead-letter': await this.getQueueStatus(this.deadLetterQueue),
+      webhooks: await this.getQueueStatus(this.webhooksQueue),
     };
   }
 
@@ -73,6 +75,7 @@ export class JobsController {
       await this.getQueueStatus(this.verificationQueue),
       await this.getQueueStatus(this.notificationsQueue),
       await this.getQueueStatus(this.onchainQueue),
+      await this.getQueueStatus(this.webhooksQueue),
     ];
 
     const isDegraded = statuses.some(s => s.waiting > 100 || s.failed > 50);

app/backend/src/jobs/jobs.module.ts

@@ -11,6 +11,7 @@ import { DlqService } from './dlq.service';
     BullModule.registerQueue({ name: 'onchain' }),
     BullModule.registerQueue({ name: RETENTION_PURGE_QUEUE }),
     BullModule.registerQueue({ name: 'dead-letter' }),
+    BullModule.registerQueue({ name: 'webhooks' }),
   ],
   controllers: [JobsController],
   providers: [DlqService],

app/backend/src/verification/verification.controller.ts

@@ -42,6 +42,7 @@ import { CreateInternalNoteDto } from 'src/common/dto/create-internal-note.dto';
 import { InternalNoteResponseDto } from 'src/common/dto/internal-note-response.dto';
 import { CacheResponse } from 'src/common/decorators/cache-response.decorator';
 import { getCacheTTL } from 'src/common/config/cache.config';
+import { WebhookService } from './webhook.service';
 
 @ApiTags('Verification')
 @ApiSecurity('x-api-key')
@@ -51,6 +52,7 @@ export class VerificationController {
     private readonly verificationService: VerificationService,
     private readonly verificationFlowService: VerificationFlowService,
     private readonly internalNotesService: InternalNotesService,
+    private readonly webhookService: WebhookService,
   ) {}
 
   @Post('claims/:id/enqueue')
@@ -510,4 +512,20 @@ export class VerificationController {
   getNotes(@Param('id') id: string) {
     return this.internalNotesService.findNotesByEntity('verification', id);
   }
+
+  @Post('webhooks/:id/replay')
+  @Version('1')
+  @ApiOperation({
+    summary: 'Replay a failed webhook delivery',
+    description: 'Resets the status of a failed webhook delivery and enqueues it for retry.',
+  })
+  @ApiOkResponse({
+    description: 'Webhook delivery queued for replay successfully.',
+  })
+  @ApiNotFoundResponse({
+    description: 'The specified webhook delivery was not found.',
+  })
+  async replayWebhook(@Param('id') id: string) {
+    return this.webhookService.replayWebhook(id);
+  }
 }

app/backend/src/verification/verification.module.ts

@@ -13,6 +13,10 @@ import { AuditModule } from '../audit/audit.module';
 import { NotificationsModule } from '../notifications/notifications.module';
 import { EncryptionModule } from '../common/encryption/encryption.module';
 import { JobsModule } from '../jobs/jobs.module';
+import { HmacModule } from '../common/hmac/hmac.module';
+import { WebhookService } from './webhook.service';
+import { WebhookProcessor } from './webhook.processor';
+import { MetricsModule } from '../observability/metrics/metrics.module';
 
 @Module({
   imports: [
@@ -22,6 +26,8 @@ import { JobsModule } from '../jobs/jobs.module';
     AuditModule,
     NotificationsModule,
     EncryptionModule,
+    HmacModule,
+    MetricsModule,
     BullModule.registerQueueAsync({
       name: 'verification',
       imports: [ConfigModule],
@@ -33,6 +39,17 @@ import { JobsModule } from '../jobs/jobs.module';
       }),
       inject: [ConfigService],
     }),
+    BullModule.registerQueueAsync({
+      name: 'webhooks',
+      imports: [ConfigModule],
+      useFactory: (configService: ConfigService) => ({
+        connection: {
+          host: configService.get<string>('REDIS_HOST') || 'localhost',
+          port: parseInt(configService.get<string>('REDIS_PORT') || '6379'),
+        },
+      }),
+      inject: [ConfigService],
+    }),
     JobsModule,
   ],
   controllers: [VerificationController, VerificationInboxController],
@@ -41,11 +58,14 @@ import { JobsModule } from '../jobs/jobs.module';
     VerificationFlowService,
     VerificationProcessor,
     VerificationInboxService,
+    WebhookService,
+    WebhookProcessor,
   ],
   exports: [
     VerificationService,
     VerificationFlowService,
     VerificationInboxService,
+    WebhookService,
   ],
 })
 export class VerificationModule {}

app/backend/src/verification/verification.service.spec.ts

@@ -8,6 +8,7 @@ import { PrismaService } from '../prisma/prisma.service';
 import { AuditService } from '../audit/audit.service';
 import { ClaimStatus, Prisma } from '@prisma/client';
 import { of } from 'rxjs';
+import { WebhookService } from './webhook.service';
 
 describe('VerificationService', () => {
   let service: VerificationService;
@@ -89,6 +90,13 @@ describe('VerificationService', () => {
             post: jest.fn().mockReturnValue(of({ data: {} })),
           },
         },
+        {
+          provide: WebhookService,
+          useValue: {
+            enqueueWebhook: jest.fn().mockResolvedValue(undefined),
+            replayWebhook: jest.fn().mockResolvedValue(undefined),
+          },
+        },
       ],
     }).compile();
 
@@ -260,6 +268,13 @@ describe('VerificationService', () => {
               post: jest.fn().mockReturnValue(of({ data: {} })),
             },
           },
+          {
+            provide: WebhookService,
+            useValue: {
+              enqueueWebhook: jest.fn().mockResolvedValue(undefined),
+              replayWebhook: jest.fn().mockResolvedValue(undefined),
+            },
+          },
         ],
       }).compile();
 

app/backend/src/verification/verification.service.ts

@@ -11,6 +11,7 @@ import { HttpService } from '@nestjs/axios';
 import { PrismaService } from '../prisma/prisma.service';
 import { ClaimStatus, Prisma } from '@prisma/client';
 import { CreateVerificationDto } from './dto/create-verification.dto';
+import { WebhookService } from './webhook.service';
 import {
   ReviewQueuePaginationMode,
   ReviewQueueQueryDto,
@@ -138,6 +139,7 @@ export class VerificationService {
     private readonly prisma: PrismaService,
     private readonly auditService: AuditService,
     private readonly httpService: HttpService,
+    private readonly webhookService: WebhookService,
   ) {
     this.verificationMode =
       this.configService.get<string>('VERIFICATION_MODE') || 'mock';
@@ -319,6 +321,18 @@ export class VerificationService {
       },
     });
 
+    try {
+      await this.webhookService.enqueueWebhook(
+        claimId,
+        shouldVerify ? 'verified' : 'requested',
+        result,
+      );
+    } catch (err) {
+      this.logger.error(
+        `Failed to enqueue webhook for claim ${claimId}: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+
     return result;
   }