Skip to content

Commit d91c876

Browse files
authored
Merge pull request #1178 from ussyalfaks/feature/timeouts-workflow-idempotency-watermark
feat(security): webhook rate limiting, timeouts, workflow idempotency
2 parents 11cfa08 + c5b7d4d commit d91c876

10 files changed

Lines changed: 337 additions & 1 deletion

File tree

backend/src/app.module.ts

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,16 @@ const envValidationSchema = Joi.object({
151151
JSON_BODY_LIMIT: Joi.string().default('1mb'),
152152
URLENCODED_BODY_LIMIT: Joi.string().default('1mb'),
153153

154+
TIMEOUT_WEBHOOK_DELIVERY_MS: Joi.number().integer().min(1000).default(10000),
155+
TIMEOUT_WEBHOOK_INGEST_MS: Joi.number().integer().min(500).default(5000),
156+
TIMEOUT_HTTP_CLIENT_MS: Joi.number().integer().min(1000).default(15000),
157+
WEBHOOK_MAX_PENDING_DELIVERIES: Joi.number().integer().min(1).default(500),
158+
WEBHOOK_INGEST_RATE_LIMIT_PER_MINUTE: Joi.number()
159+
.integer()
160+
.min(1)
161+
.default(30),
162+
WEBHOOK_MAX_INGEST_QUEUE_DEPTH: Joi.number().integer().min(1).default(1000),
163+
154164
IDEMPOTENCY_CLEANUP_ENABLED: Joi.boolean().default(true),
155165
IDEMPOTENCY_CLEANUP_CRON: Joi.string()
156166
.regex(/^(\S+\s+){4}\S+$/)
@@ -414,6 +424,15 @@ const envValidationSchema = Joi.object({
414424
ttl: 5 * 60 * 1000, // 5 minutes
415425
limit: 2,
416426
},
427+
{
428+
// Inbound webhook ingest (e.g. /webhooks/stellar).
429+
// 30 requests per minute per sender/IP — tight enough to defeat
430+
// spam floods but generous enough to allow legitimate bursts.
431+
// Tune via WEBHOOK_INGEST_RATE_LIMIT_PER_MINUTE env var.
432+
name: 'webhook-ingest',
433+
ttl: 60_000, // 1 minute
434+
limit: 30,
435+
},
417436
]),
418437
],
419438
controllers: [AppController],

backend/src/common/common.module.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import { DataScopeService } from './services/data-scope.service';
2121
import { DistributedLockModule } from './distributed-lock/distributed-lock.module';
2222
import { TestModeModule } from './test-mode/test-mode.module';
2323
import { EventualConsistencyService } from './services/eventual-consistency.service';
24+
import { WorkflowIdempotencyService } from './services/workflow-idempotency.service';
2425

2526
@Global()
2627
@Module({
@@ -46,6 +47,7 @@ import { EventualConsistencyService } from './services/eventual-consistency.serv
4647
TenantContextMiddleware,
4748
DataScopeService,
4849
EventualConsistencyService,
50+
WorkflowIdempotencyService,
4951
],
5052
exports: [
5153
RateLimitMonitorService,
@@ -62,6 +64,7 @@ import { EventualConsistencyService } from './services/eventual-consistency.serv
6264
DataScopeService,
6365
DistributedLockModule,
6466
EventualConsistencyService,
67+
WorkflowIdempotencyService,
6568
],
6669
})
6770
export class CommonModule {}

backend/src/common/guards/tiered-throttler.guard.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ const TIER_LIMITS: Record<
4141
vote: { limit: 5, ttl: 60_000 },
4242
upload: { limit: 10, ttl: 60_000 },
4343
'admin-high-risk': { limit: 0, ttl: 5 * 60 * 1000 }, // Not allowed
44+
'webhook-ingest': { limit: 30, ttl: 60_000 },
4445
},
4546
[UserTier.VERIFIED]: {
4647
default: { limit: 150, ttl: 60000 },
@@ -52,6 +53,7 @@ const TIER_LIMITS: Record<
5253
vote: { limit: 10, ttl: 60_000 },
5354
upload: { limit: 30, ttl: 60_000 },
5455
'admin-high-risk': { limit: 0, ttl: 5 * 60 * 1000 }, // Not allowed
56+
'webhook-ingest': { limit: 60, ttl: 60_000 },
5557
},
5658
[UserTier.PREMIUM]: {
5759
default: { limit: 300, ttl: 60000 },
@@ -63,6 +65,7 @@ const TIER_LIMITS: Record<
6365
vote: { limit: 20, ttl: 60_000 },
6466
upload: { limit: 60, ttl: 60_000 },
6567
'admin-high-risk': { limit: 0, ttl: 5 * 60 * 1000 }, // Not allowed
68+
'webhook-ingest': { limit: 120, ttl: 60_000 },
6669
},
6770
[UserTier.ENTERPRISE]: {
6871
default: { limit: 1000, ttl: 60000 },
@@ -74,6 +77,7 @@ const TIER_LIMITS: Record<
7477
vote: { limit: 30, ttl: 60_000 },
7578
upload: { limit: 120, ttl: 60_000 },
7679
'admin-high-risk': { limit: 0, ttl: 5 * 60 * 1000 }, // Not allowed
80+
'webhook-ingest': { limit: 300, ttl: 60_000 },
7781
},
7882
[UserTier.ADMIN]: {
7983
default: { limit: 1000, ttl: 60000 },
@@ -85,6 +89,7 @@ const TIER_LIMITS: Record<
8589
vote: { limit: 30, ttl: 60_000 },
8690
upload: { limit: 240, ttl: 60_000 },
8791
'admin-high-risk': { limit: 2, ttl: 5 * 60 * 1000 }, // 2 per 5 minutes
92+
'webhook-ingest': { limit: 600, ttl: 60_000 },
8893
},
8994
};
9095

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import { Injectable, Inject, Logger } from '@nestjs/common';
2+
import { CACHE_MANAGER } from '@nestjs/cache-manager';
3+
import { Cache } from 'cache-manager';
4+
5+
export interface WorkflowStepRecord {
6+
completedAt: string;
7+
result?: unknown;
8+
}
9+
10+
export interface WorkflowState {
11+
workflowId: string;
12+
steps: Record<string, WorkflowStepRecord>;
13+
startedAt: string;
14+
}
15+
16+
const WORKFLOW_KEY_PREFIX = 'workflow-idempotency';
17+
/** Default TTL: 7 days — long enough to cover asynchronous multi-step retry windows. */
18+
const DEFAULT_TTL_MS = 7 * 24 * 60 * 60 * 1000;
19+
20+
/**
21+
* WorkflowIdempotencyService
22+
*
23+
* Provides step-level idempotency for multi-step workflows. Each workflow
24+
* is identified by a `workflowId`. Individual steps within the workflow
25+
* are identified by a `stepName`. On retry, callers can skip already-
26+
* completed steps rather than re-executing them.
27+
*
28+
* Usage pattern:
29+
*
30+
* const wf = workflowIdempotencyService;
31+
* const wfId = `deposit-${idempotencyKey}`;
32+
*
33+
* if (!(await wf.isStepCompleted(wfId, 'validate'))) {
34+
* await validateDeposit(payload);
35+
* await wf.markStepCompleted(wfId, 'validate');
36+
* }
37+
*
38+
* if (!(await wf.isStepCompleted(wfId, 'reserve'))) {
39+
* const reservation = await reserveFunds(payload);
40+
* await wf.markStepCompleted(wfId, 'reserve', { reservationId: reservation.id });
41+
* }
42+
*
43+
* // ... further steps
44+
* await wf.clearWorkflow(wfId); // optional cleanup after success
45+
*
46+
* The workflow state is stored in the shared cache (Redis in production,
47+
* in-memory in tests) under the key `workflow-idempotency:<workflowId>`.
48+
*/
49+
@Injectable()
50+
export class WorkflowIdempotencyService {
51+
private readonly logger = new Logger(WorkflowIdempotencyService.name);
52+
53+
constructor(@Inject(CACHE_MANAGER) private readonly cache: Cache) {}
54+
55+
/**
56+
* Returns true if the given step has already been completed for the
57+
* specified workflow. Callers should skip execution of the step when
58+
* this returns true.
59+
*/
60+
async isStepCompleted(
61+
workflowId: string,
62+
stepName: string,
63+
): Promise<boolean> {
64+
const state = await this.getWorkflowState(workflowId);
65+
return state !== null && stepName in state.steps;
66+
}
67+
68+
/**
69+
* Marks a workflow step as completed, optionally persisting a small
70+
* result payload (e.g. an ID or status) for downstream steps to read.
71+
*
72+
* @param workflowId Unique identifier for the workflow execution.
73+
* @param stepName Name of the step being marked complete.
74+
* @param result Optional serialisable result to store with the step.
75+
* @param ttlMs Time-to-live for the whole workflow record in ms.
76+
*/
77+
async markStepCompleted(
78+
workflowId: string,
79+
stepName: string,
80+
result?: unknown,
81+
ttlMs = DEFAULT_TTL_MS,
82+
): Promise<void> {
83+
const state = (await this.getWorkflowState(workflowId)) ?? {
84+
workflowId,
85+
steps: {},
86+
startedAt: new Date().toISOString(),
87+
};
88+
89+
state.steps[stepName] = {
90+
completedAt: new Date().toISOString(),
91+
result,
92+
};
93+
94+
await this.cache.set(this.cacheKey(workflowId), state, ttlMs);
95+
this.logger.debug(
96+
`Workflow ${workflowId}: step "${stepName}" marked completed`,
97+
);
98+
}
99+
100+
/**
101+
* Returns the stored result for a previously completed step, or
102+
* `undefined` if the step has not been completed or stored no result.
103+
*/
104+
async getStepResult<T = unknown>(
105+
workflowId: string,
106+
stepName: string,
107+
): Promise<T | undefined> {
108+
const state = await this.getWorkflowState(workflowId);
109+
return state?.steps[stepName]?.result as T | undefined;
110+
}
111+
112+
/**
113+
* Returns the full workflow state, or null if no state is stored for
114+
* the given workflowId.
115+
*/
116+
async getWorkflowState(workflowId: string): Promise<WorkflowState | null> {
117+
return (
118+
((await this.cache.get(this.cacheKey(workflowId))) as WorkflowState) ??
119+
null
120+
);
121+
}
122+
123+
/**
124+
* Returns the set of step names that have been completed for the given
125+
* workflow, or an empty array if the workflow has no stored state.
126+
*/
127+
async getCompletedSteps(workflowId: string): Promise<string[]> {
128+
const state = await this.getWorkflowState(workflowId);
129+
return state ? Object.keys(state.steps) : [];
130+
}
131+
132+
/**
133+
* Removes all stored state for a workflow. Call this after a workflow
134+
* completes successfully to free cache space; the TTL will clean it up
135+
* automatically if this is not called.
136+
*/
137+
async clearWorkflow(workflowId: string): Promise<void> {
138+
await this.cache.del(this.cacheKey(workflowId));
139+
this.logger.debug(`Workflow ${workflowId}: state cleared`);
140+
}
141+
142+
private cacheKey(workflowId: string): string {
143+
return `${WORKFLOW_KEY_PREFIX}:${workflowId}`;
144+
}
145+
}

backend/src/config/configuration.ts

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,33 @@ export default () => ({
101101
),
102102
backoffDelay: parseInt(process.env.JOB_QUEUE_BACKOFF_DELAY || '2000', 10),
103103
},
104+
webhook: {
105+
/**
106+
* Maximum number of PENDING outbound deliveries allowed before
107+
* new dispatch calls are shed (backpressure). Default: 500.
108+
*/
109+
maxPendingDeliveries: parseInt(
110+
process.env.WEBHOOK_MAX_PENDING_DELIVERIES || '500',
111+
10,
112+
),
113+
/**
114+
* Maximum inbound webhook requests per sender/IP per minute before
115+
* the rate-limiter rejects with 429. Default: 30.
116+
*/
117+
ingestRateLimitPerMinute: parseInt(
118+
process.env.WEBHOOK_INGEST_RATE_LIMIT_PER_MINUTE || '30',
119+
10,
120+
),
121+
/**
122+
* Maximum number of PENDING outbound deliveries allowed before the
123+
* WebhookIngestGuard rejects inbound requests with 503 (backpressure).
124+
* Default: 1000.
125+
*/
126+
maxIngestQueueDepth: parseInt(
127+
process.env.WEBHOOK_MAX_INGEST_QUEUE_DEPTH || '1000',
128+
10,
129+
),
130+
},
104131
mail: {
105132
host: process.env.MAIL_HOST,
106133
port: parseInt(process.env.MAIL_PORT || '587', 10),
@@ -243,6 +270,35 @@ export default () => ({
243270
10,
244271
),
245272
},
273+
timeouts: {
274+
/**
275+
* Outbound webhook delivery — how long we wait for a subscriber's
276+
* endpoint to respond before treating the delivery as failed.
277+
* Default: 10 s. Tune per-environment via TIMEOUT_WEBHOOK_DELIVERY_MS.
278+
*/
279+
webhookDeliveryMs: parseInt(
280+
process.env.TIMEOUT_WEBHOOK_DELIVERY_MS || '10000',
281+
10,
282+
),
283+
/**
284+
* Inbound webhook ingest — maximum time (ms) the verification
285+
* middleware + controller handler may run before the request is
286+
* aborted. Protects against slow-loris or stalled DB calls on the
287+
* ingest path. Default: 5 s.
288+
*/
289+
webhookIngestMs: parseInt(
290+
process.env.TIMEOUT_WEBHOOK_INGEST_MS || '5000',
291+
10,
292+
),
293+
/**
294+
* Default HTTP client timeout used by any service that does not
295+
* have an explicit per-endpoint override. Default: 15 s.
296+
*/
297+
httpClientMs: parseInt(
298+
process.env.TIMEOUT_HTTP_CLIENT_MS || '15000',
299+
10,
300+
),
301+
},
246302
blockchainReplay: {
247303
maxLedgerRange: parseInt(
248304
process.env.BLOCKCHAIN_REPLAY_MAX_LEDGER_RANGE || '10000',

backend/src/modules/data-export/data-export.service.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,15 @@ export class DataExportService {
288288
]);
289289

290290
const zipPath = path.join(EXPORT_DIR, `${request.id}.zip`);
291+
const exportedAt = new Date().toISOString();
291292
await this.buildZip(zipPath, {
293+
'WATERMARK.json': {
294+
userId: user.id,
295+
requestId: request.id,
296+
exportedAt,
297+
_notice:
298+
'This export was generated exclusively for the user identified above. Unauthorized redistribution is prohibited.',
299+
},
292300
'profile.json': {
293301
id: user.id,
294302
email: user.email,
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import {
2+
CanActivate,
3+
ExecutionContext,
4+
Injectable,
5+
Logger,
6+
ServiceUnavailableException,
7+
} from '@nestjs/common';
8+
import { ConfigService } from '@nestjs/config';
9+
import { InjectRepository } from '@nestjs/typeorm';
10+
import { Repository } from 'typeorm';
11+
import { Response } from 'express';
12+
import {
13+
WebhookDelivery,
14+
DeliveryStatus,
15+
} from '../entities/webhook-delivery.entity';
16+
17+
const DEFAULT_MAX_INGEST_QUEUE_DEPTH = 1000;
18+
const BACKPRESSURE_RETRY_AFTER_SECONDS = 30;
19+
20+
/**
21+
* Guards the inbound webhook ingest endpoint against queue overload.
22+
*
23+
* Before admitting a new inbound request, counts the number of PENDING
24+
* outbound deliveries. If that count meets or exceeds the configured
25+
* maximum, the request is rejected with HTTP 503 and a `Retry-After`
26+
* header so well-behaved senders back off.
27+
*
28+
* This implements the "backpressure when ingest is overloaded" requirement
29+
* from #1168. The queue-depth threshold is tunable via
30+
* `WEBHOOK_MAX_INGEST_QUEUE_DEPTH` (default 1000).
31+
*/
32+
@Injectable()
33+
export class WebhookIngestGuard implements CanActivate {
34+
private readonly logger = new Logger(WebhookIngestGuard.name);
35+
36+
constructor(
37+
@InjectRepository(WebhookDelivery)
38+
private readonly deliveryRepo: Repository<WebhookDelivery>,
39+
private readonly configService: ConfigService,
40+
) {}
41+
42+
async canActivate(context: ExecutionContext): Promise<boolean> {
43+
const maxQueueDepth =
44+
this.configService.get<number>('webhook.maxIngestQueueDepth') ??
45+
DEFAULT_MAX_INGEST_QUEUE_DEPTH;
46+
47+
const pendingCount = await this.deliveryRepo.count({
48+
where: { status: DeliveryStatus.PENDING },
49+
});
50+
51+
if (pendingCount >= maxQueueDepth) {
52+
const response = context.switchToHttp().getResponse<Response>();
53+
response.setHeader('Retry-After', BACKPRESSURE_RETRY_AFTER_SECONDS);
54+
55+
this.logger.warn(
56+
`Webhook ingest backpressure: ${pendingCount}/${maxQueueDepth} pending deliveries. Rejecting inbound request.`,
57+
);
58+
59+
throw new ServiceUnavailableException({
60+
message:
61+
'Webhook ingest is at capacity. Please retry after the indicated interval.',
62+
retryAfterSeconds: BACKPRESSURE_RETRY_AFTER_SECONDS,
63+
});
64+
}
65+
66+
return true;
67+
}
68+
}

0 commit comments

Comments
 (0)