diff --git a/lambdas/functions/runner-count-cache/package.json b/lambdas/functions/runner-count-cache/package.json new file mode 100644 index 0000000000..ea173203d4 --- /dev/null +++ b/lambdas/functions/runner-count-cache/package.json @@ -0,0 +1,40 @@ +{ + "name": "@aws-github-runner/runner-count-cache", + "version": "1.0.0", + "main": "lambda.ts", + "type": "module", + "license": "MIT", + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "build": "ncc build src/lambda.ts -o dist", + "dist": "yarn build && cp package.json dist/ && cd dist && zip ../runner-count-cache.zip *", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn build && yarn format && yarn lint && yarn test" + }, + "devDependencies": { + "@aws-sdk/types": "^3.936.0", + "@types/aws-lambda": "^8.10.155", + "@types/node": "^22.19.0", + "@vercel/ncc": "^0.38.4", + "aws-sdk-client-mock": "^4.1.0", + "aws-sdk-client-mock-jest": "^4.1.0" + }, + "dependencies": { + "@aws-github-runner/aws-powertools-util": "*", + "@aws-sdk/client-dynamodb": "^3.948.0", + "@aws-sdk/client-ec2": "^3.948.0" + }, + "nx": { + "includedScripts": [ + "build", + "dist", + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/functions/runner-count-cache/src/lambda.test.ts b/lambdas/functions/runner-count-cache/src/lambda.test.ts new file mode 100644 index 0000000000..03c0d670c4 --- /dev/null +++ b/lambdas/functions/runner-count-cache/src/lambda.test.ts @@ -0,0 +1,129 @@ +import { EC2Client, DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { DynamoDBClient, TransactWriteItemsCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import { beforeEach, describe, expect, it } from 'vitest'; +import type { Context } from 'aws-lambda'; + +import { handler } from './lambda'; + +const ec2Mock = mockClient(EC2Client); +const ddbMock = mockClient(DynamoDBClient); + +const ctx = { awsRequestId: 'test', functionName: 'runner-count-cache' } as unknown as Context; + +function event(instanceId: string, state: string) { + // Only event.detail is read by the handler. + return { detail: { 'instance-id': instanceId, state } } as never; +} + +function runnerInstance(tags: { Key: string; Value: string }[]) { + return { Reservations: [{ Instances: [{ Tags: tags }] }] }; +} + +const defaultTags = [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:environment', Value: 'prod' }, + { Key: 'ghr:Type', Value: 'Repo' }, + { Key: 'ghr:Owner', Value: 'acme/app' }, +]; + +const transactions = () => ddbMock.commandCalls(TransactWriteItemsCommand).map((c) => c.args[0].input); +const condFail = () => Object.assign(new Error('cancelled'), { name: 'TransactionCanceledException' }); + +beforeEach(() => { + ec2Mock.reset(); + ddbMock.reset(); + process.env.AWS_REGION = 'eu-west-1'; + process.env.DYNAMODB_TABLE_NAME = 'runner-count'; + delete process.env.ENVIRONMENT_FILTER; + ec2Mock.on(DescribeInstancesCommand).resolves(runnerInstance(defaultTags)); + ddbMock.on(TransactWriteItemsCommand).resolves({}); +}); + +describe('runner-count-cache handler', () => { + it('pending: creates marker (if absent) and increments, atomically', async () => { + await handler(event('i-1', 'pending'), ctx); + const tx = transactions(); + expect(tx).toHaveLength(1); + const items = tx[0].TransactItems!; + expect(items[0].Put!.ConditionExpression).toBe('attribute_not_exists(pk)'); + expect((items[0].Put!.Item!.pk as { S: string }).S).toBe('INSTANCE#i-1'); + expect((items[0].Put!.Item!.state as { S: string }).S).toBe('COUNTED'); + expect(items[1].Update!.Key!.pk).toEqual({ S: 'prod#Repo#acme/app' }); + expect(items[1].Update!.ExpressionAttributeValues![':one']).toEqual({ N: '1' }); + }); + + it('running: also attempts +1 (marker dedups the pending->running pair)', async () => { + await handler(event('i-1', 'running'), ctx); + expect(transactions()).toHaveLength(1); + expect(transactions()[0].TransactItems![0].Put!.ConditionExpression).toBe('attribute_not_exists(pk)'); + }); + + it('duplicate active event is an idempotent no-op (transaction cancelled)', async () => { + ddbMock.on(TransactWriteItemsCommand).rejects(condFail()); + await expect(handler(event('i-1', 'running'), ctx)).resolves.toBeUndefined(); + }); + + it('terminated: transitions marker COUNTED->TERMINATED and decrements', async () => { + await handler(event('i-1', 'terminated'), ctx); + const items = transactions()[0].TransactItems!; + expect(items[0].Update!.ConditionExpression).toBe('attribute_exists(pk) AND #state = :counted'); + expect(items[0].Update!.ExpressionAttributeValues![':term']).toEqual({ S: 'TERMINATED' }); + expect(items[1].Update!.ExpressionAttributeValues![':negOne']).toEqual({ N: '-1' }); + }); + + it('duplicate terminated is an idempotent no-op (no double decrement)', async () => { + ddbMock.on(TransactWriteItemsCommand).rejects(condFail()); + await expect(handler(event('i-1', 'terminated'), ctx)).resolves.toBeUndefined(); + }); + + it('terminated for an uncounted instance does not underflow (guard rejects)', async () => { + // marker missing / not COUNTED -> transaction cancelled -> no decrement + ddbMock.on(TransactWriteItemsCommand).rejects(condFail()); + await handler(event('i-unknown', 'terminated'), ctx); + // handler swallowed the cancellation; the -1 never landed + await expect(handler(event('i-unknown', 'terminated'), ctx)).resolves.toBeUndefined(); + }); + + it('ignores non-runner instances', async () => { + ec2Mock + .on(DescribeInstancesCommand) + .resolves(runnerInstance([{ Key: 'ghr:Application', Value: 'something-else' }])); + await handler(event('i-1', 'pending'), ctx); + expect(transactions()).toHaveLength(0); + }); + + it('respects the environment filter', async () => { + process.env.ENVIRONMENT_FILTER = 'prod'; + ec2Mock.on(DescribeInstancesCommand).resolves( + runnerInstance([ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:environment', Value: 'staging' }, + { Key: 'ghr:Type', Value: 'Repo' }, + { Key: 'ghr:Owner', Value: 'acme/app' }, + ]), + ); + await handler(event('i-1', 'pending'), ctx); + expect(transactions()).toHaveLength(0); + }); + + it('skips instances missing required tags', async () => { + ec2Mock + .on(DescribeInstancesCommand) + .resolves(runnerInstance([{ Key: 'ghr:Application', Value: 'github-action-runner' }])); + await handler(event('i-1', 'pending'), ctx); + expect(transactions()).toHaveLength(0); + }); + + it('does not touch DynamoDB for transitional states (e.g. stopping->? unmapped)', async () => { + await handler(event('i-1', 'rebooting'), ctx); + expect(transactions()).toHaveLength(0); + }); + + it('rethrows non-cancellation errors so EventBridge retries / DLQs', async () => { + ddbMock + .on(TransactWriteItemsCommand) + .rejects(Object.assign(new Error('throttled'), { name: 'ProvisionedThroughputExceededException' })); + await expect(handler(event('i-1', 'pending'), ctx)).rejects.toThrow('throttled'); + }); +}); diff --git a/lambdas/functions/runner-count-cache/src/lambda.ts b/lambdas/functions/runner-count-cache/src/lambda.ts new file mode 100644 index 0000000000..0547f62e62 --- /dev/null +++ b/lambdas/functions/runner-count-cache/src/lambda.ts @@ -0,0 +1,259 @@ +/** + * Runner Count Cache Lambda + * + * This Lambda function is triggered by EventBridge when EC2 instances change state. + * It updates an atomic counter in DynamoDB to track the number of active runners + * per environment/type/owner combination. + * + * This eliminates the need for repeated DescribeInstances API calls during scale-up, + * addressing the performance bottleneck described in Issue #4710. + * + * @see https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710 + */ + +import { EventBridgeEvent, Context } from 'aws-lambda'; +import { DynamoDBClient, TransactWriteItemsCommand } from '@aws-sdk/client-dynamodb'; +import { EC2Client, DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { createChildLogger, setContext } from '@aws-github-runner/aws-powertools-util'; + +const logger = createChildLogger('runner-count-cache'); + +interface EC2StateChangeDetail { + 'instance-id': string; + state: 'pending' | 'running' | 'shutting-down' | 'stopped' | 'stopping' | 'terminated'; +} + +interface InstanceTags { + environment?: string; + type?: string; + owner?: string; + application?: string; +} + +/** + * Get instance tags from EC2 to determine if this is a managed runner + */ +async function getInstanceTags(ec2: EC2Client, instanceId: string): Promise { + try { + const result = await ec2.send( + new DescribeInstancesCommand({ + InstanceIds: [instanceId], + }), + ); + + const instance = result.Reservations?.[0]?.Instances?.[0]; + if (!instance) { + logger.debug('Instance not found', { instanceId }); + return null; + } + + const tags = instance.Tags || []; + return { + environment: tags.find((t) => t.Key === 'ghr:environment')?.Value, + type: tags.find((t) => t.Key === 'ghr:Type')?.Value, + owner: tags.find((t) => t.Key === 'ghr:Owner')?.Value, + application: tags.find((t) => t.Key === 'ghr:Application')?.Value, + }; + } catch (error) { + // Instance might already be terminated, which is fine + logger.debug('Failed to get instance tags', { instanceId, error }); + return null; + } +} + +/** + * A COUNTED marker must outlive the runner so a late-redelivered `running` + * cannot re-increment. Terminated markers are retained (via the shorter cleanup + * TTL) only long enough to dedup late `terminated` redeliveries, then swept. + */ +const COUNTED_MARKER_TTL_SECONDS = 7 * 86400; // 7 days + +/** + * Idempotent +1. Atomically creates the per-instance marker only if it does not + * already exist, and increments the group counter. EventBridge is at-least-once + * and unordered, so a redelivered `pending`/`running` finds the marker present + * and the transaction is cancelled — no double count, and no IDLE->LAUNCHING style + * regression. + */ +async function countInstance( + dynamodb: DynamoDBClient, + tableName: string, + counterPk: string, + instanceId: string, + ttlSeconds: number, +): Promise { + const now = Date.now(); + const nowSec = Math.floor(now / 1000); + await dynamodb.send( + new TransactWriteItemsCommand({ + TransactItems: [ + { + Put: { + TableName: tableName, + Item: { + pk: { S: `INSTANCE#${instanceId}` }, + state: { S: 'COUNTED' }, + updated: { N: String(now) }, + ttl: { N: String(nowSec + COUNTED_MARKER_TTL_SECONDS) }, + }, + ConditionExpression: 'attribute_not_exists(pk)', + }, + }, + { + Update: { + TableName: tableName, + Key: { pk: { S: counterPk } }, + UpdateExpression: 'ADD #count :one SET #updated = :now, #ttl = :ttl', + ExpressionAttributeNames: { '#count': 'count', '#updated': 'updated', '#ttl': 'ttl' }, + ExpressionAttributeValues: { + ':one': { N: '1' }, + ':now': { N: String(now) }, + ':ttl': { N: String(nowSec + ttlSeconds) }, + }, + }, + }, + ], + }), + ); +} + +/** + * Idempotent -1. Atomically transitions the marker COUNTED -> TERMINATED only if + * it is currently COUNTED, and decrements the group counter. A redelivered + * `terminated`, or one for an instance this system never counted (out-of-order, + * or pre-existing before the feature was enabled), fails the condition and the + * transaction is cancelled — no double decrement, and no negative drift. The + * marker guard is what makes a separate write-time floor unnecessary: a -1 can + * only apply when the matching +1 was recorded here. + */ +async function uncountInstance( + dynamodb: DynamoDBClient, + tableName: string, + counterPk: string, + instanceId: string, + ttlSeconds: number, +): Promise { + const now = Date.now(); + const nowSec = Math.floor(now / 1000); + await dynamodb.send( + new TransactWriteItemsCommand({ + TransactItems: [ + { + Update: { + TableName: tableName, + Key: { pk: { S: `INSTANCE#${instanceId}` } }, + UpdateExpression: 'SET #state = :term, #updated = :now, #ttl = :ttl', + ConditionExpression: 'attribute_exists(pk) AND #state = :counted', + ExpressionAttributeNames: { '#state': 'state', '#updated': 'updated', '#ttl': 'ttl' }, + ExpressionAttributeValues: { + ':term': { S: 'TERMINATED' }, + ':counted': { S: 'COUNTED' }, + ':now': { N: String(now) }, + ':ttl': { N: String(nowSec + ttlSeconds) }, + }, + }, + }, + { + Update: { + TableName: tableName, + Key: { pk: { S: counterPk } }, + UpdateExpression: 'ADD #count :negOne SET #updated = :now', + ExpressionAttributeNames: { '#count': 'count', '#updated': 'updated' }, + ExpressionAttributeValues: { ':negOne': { N: '-1' }, ':now': { N: String(now) } }, + }, + }, + ], + }), + ); +} + +/** + * Lambda handler for EC2 state change events + */ +export async function handler( + event: EventBridgeEvent<'EC2 Instance State-change Notification', EC2StateChangeDetail>, + context: Context, +): Promise { + setContext(context, 'lambda.ts'); + + const instanceId = event.detail['instance-id']; + const state = event.detail.state; + const tableName = process.env.DYNAMODB_TABLE_NAME; + const environmentFilter = process.env.ENVIRONMENT_FILTER; + const ttlSeconds = parseInt(process.env.TTL_SECONDS || '86400', 10); + + if (!tableName) { + logger.error('DYNAMODB_TABLE_NAME environment variable not set'); + return; + } + + logger.info('Processing EC2 state change', { instanceId, state }); + + const ec2 = new EC2Client({ region: process.env.AWS_REGION }); + const dynamodb = new DynamoDBClient({ region: process.env.AWS_REGION }); + + // Get instance tags to check if this is a managed runner + const tags = await getInstanceTags(ec2, instanceId); + + if (!tags) { + logger.debug('Could not get instance tags, skipping', { instanceId }); + return; + } + + // Check if this is a GitHub Action runner + if (tags.application !== 'github-action-runner') { + logger.debug('Instance is not a GitHub Action runner, skipping', { instanceId }); + return; + } + + // Check if environment matches our filter + if (environmentFilter && tags.environment !== environmentFilter) { + logger.debug('Instance environment does not match filter, skipping', { + instanceId, + instanceEnv: tags.environment, + filterEnv: environmentFilter, + }); + return; + } + + // Ensure we have required tags + if (!tags.environment || !tags.type || !tags.owner) { + logger.debug('Instance missing required tags, skipping', { instanceId, tags }); + return; + } + + // Generate partition key + const pk = `${tags.environment}#${tags.type}#${tags.owner}`; + + // Map state to a counting intent. With the per-instance marker guard we can + // safely count on the first "active" event (pending or running): whichever + // arrives first records the marker and increments, and the other is a no-op. + // This avoids the pending->running double-count without missing a runner that + // dies before ever reaching `running`. + const isActive = state === 'pending' || state === 'running'; + const isGone = state === 'terminated' || state === 'stopped' || state === 'shutting-down'; + + if (!isActive && !isGone) { + logger.debug('State does not affect counter', { state }); + return; + } + + try { + if (isActive) { + await countInstance(dynamodb, tableName, pk, instanceId, ttlSeconds); + } else { + await uncountInstance(dynamodb, tableName, pk, instanceId, ttlSeconds); + } + logger.info('Counter updated', { pk, state }); + } catch (error) { + // A cancelled transaction means the marker guard rejected the write: the + // event is a duplicate or out-of-order and has already been accounted for. + // That is an expected idempotent no-op, not a failure. + if ((error as { name?: string }).name === 'TransactionCanceledException') { + logger.debug('Duplicate/out-of-order event ignored (idempotent no-op)', { pk, instanceId, state }); + return; + } + logger.error('Failed to update counter', { pk, state, error }); + throw error; + } +} diff --git a/lambdas/functions/runner-count-cache/tsconfig.json b/lambdas/functions/runner-count-cache/tsconfig.json new file mode 100644 index 0000000000..30cbbee83e --- /dev/null +++ b/lambdas/functions/runner-count-cache/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends" : "../../tsconfig.json", + "include": [ + "src/**/*" + ], + "exclude": [ + "src/**/*.test.ts" + ] +} diff --git a/lambdas/functions/runner-count-cache/vitest.config.ts b/lambdas/functions/runner-count-cache/vitest.config.ts new file mode 100644 index 0000000000..2bb87ce333 --- /dev/null +++ b/lambdas/functions/runner-count-cache/vitest.config.ts @@ -0,0 +1,20 @@ +import { resolve } from 'path'; + +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + setupFiles: [resolve(__dirname, '../../aws-vitest-setup.ts')], + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/*.d.ts'], + thresholds: { + statements: 90, + branches: 90, + functions: 90, + lines: 90, + }, + }, + }, +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.test.ts new file mode 100644 index 0000000000..45d736e33d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.test.ts @@ -0,0 +1,239 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import { ec2RunnerCountCache, dynamoDbRunnerCountCache } from './runner-count-cache'; +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; + +const mockDynamoDBClient = mockClient(DynamoDBClient); + +describe('ec2RunnerCountCache', () => { + beforeEach(() => { + ec2RunnerCountCache.reset(); + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + describe('get', () => { + it('should return undefined when cache is empty', () => { + const result = ec2RunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBeUndefined(); + }); + + it('should return cached value when within TTL', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + + // Advance time by 3 seconds (within default 5s TTL) + vi.advanceTimersByTime(3000); + + const result = ec2RunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBe(10); + }); + + it('should return undefined when cache entry is expired', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + + // Advance time by 6 seconds (past default 5s TTL) + vi.advanceTimersByTime(6000); + + const result = ec2RunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBeUndefined(); + }); + + it('should respect custom TTL', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + + // Advance time by 8 seconds + vi.advanceTimersByTime(8000); + + // Should be expired with default TTL but valid with custom 10s TTL + const expiredResult = ec2RunnerCountCache.get('prod', 'Org', 'my-org', 5000); + expect(expiredResult).toBeUndefined(); + + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 15); + vi.advanceTimersByTime(8000); + + const validResult = ec2RunnerCountCache.get('prod', 'Org', 'my-org', 10000); + expect(validResult).toBe(15); + }); + + it('should return different values for different keys', () => { + ec2RunnerCountCache.set('prod', 'Org', 'org-a', 10); + ec2RunnerCountCache.set('prod', 'Org', 'org-b', 20); + ec2RunnerCountCache.set('prod', 'Repo', 'owner/repo', 5); + + expect(ec2RunnerCountCache.get('prod', 'Org', 'org-a')).toBe(10); + expect(ec2RunnerCountCache.get('prod', 'Org', 'org-b')).toBe(20); + expect(ec2RunnerCountCache.get('prod', 'Repo', 'owner/repo')).toBe(5); + }); + }); + + describe('set', () => { + it('should store value in cache', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBe(10); + }); + + it('should overwrite existing value', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 20); + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBe(20); + }); + }); + + describe('increment', () => { + it('should increment existing cached value', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + ec2RunnerCountCache.increment('prod', 'Org', 'my-org', 5); + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBe(15); + }); + + it('should handle negative increments (decrement)', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + ec2RunnerCountCache.increment('prod', 'Org', 'my-org', -3); + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBe(7); + }); + + it('should do nothing if cache entry does not exist', () => { + ec2RunnerCountCache.increment('prod', 'Org', 'my-org', 5); + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBeUndefined(); + }); + + it('should reset TTL on increment', () => { + ec2RunnerCountCache.set('prod', 'Org', 'my-org', 10); + + // Advance time by 4 seconds + vi.advanceTimersByTime(4000); + + // Increment, which should reset the TTL + ec2RunnerCountCache.increment('prod', 'Org', 'my-org', 1); + + // Advance another 4 seconds (total 8 seconds from original set, but only 4 from increment) + vi.advanceTimersByTime(4000); + + // Should still be valid because TTL was reset + expect(ec2RunnerCountCache.get('prod', 'Org', 'my-org')).toBe(11); + }); + }); + + describe('reset', () => { + it('should clear all cache entries', () => { + ec2RunnerCountCache.set('prod', 'Org', 'org-a', 10); + ec2RunnerCountCache.set('prod', 'Org', 'org-b', 20); + + expect(ec2RunnerCountCache.size()).toBe(2); + + ec2RunnerCountCache.reset(); + + expect(ec2RunnerCountCache.size()).toBe(0); + expect(ec2RunnerCountCache.get('prod', 'Org', 'org-a')).toBeUndefined(); + }); + }); + + describe('size', () => { + it('should return correct cache size', () => { + expect(ec2RunnerCountCache.size()).toBe(0); + + ec2RunnerCountCache.set('prod', 'Org', 'org-a', 10); + expect(ec2RunnerCountCache.size()).toBe(1); + + ec2RunnerCountCache.set('prod', 'Org', 'org-b', 20); + expect(ec2RunnerCountCache.size()).toBe(2); + }); + }); +}); + +describe('dynamoDbRunnerCountCache', () => { + beforeEach(() => { + dynamoDbRunnerCountCache.reset(); + mockDynamoDBClient.reset(); + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + describe('isEnabled', () => { + it('should return false when not initialized', () => { + expect(dynamoDbRunnerCountCache.isEnabled()).toBe(false); + }); + + it('should return true after initialization', () => { + dynamoDbRunnerCountCache.initialize('test-table', 'us-east-1', 60000); + expect(dynamoDbRunnerCountCache.isEnabled()).toBe(true); + }); + }); + + describe('get', () => { + beforeEach(() => { + dynamoDbRunnerCountCache.initialize('test-table', 'us-east-1', 60000); + }); + + it('should return null when item not found in DynamoDB', async () => { + mockDynamoDBClient.on(GetItemCommand).resolves({ + Item: undefined, + }); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBeNull(); + }); + + it('should return count and isStale=false when item is fresh', async () => { + const now = Date.now(); + mockDynamoDBClient.on(GetItemCommand).resolves({ + Item: { + pk: { S: 'prod#Org#my-org' }, + count: { N: '10' }, + updated: { N: String(now - 30000) }, // 30 seconds ago + }, + }); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toEqual({ count: 10, isStale: false }); + }); + + it('should return count and isStale=true when item is stale', async () => { + const now = Date.now(); + mockDynamoDBClient.on(GetItemCommand).resolves({ + Item: { + pk: { S: 'prod#Org#my-org' }, + count: { N: '10' }, + updated: { N: String(now - 120000) }, // 2 minutes ago + }, + }); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toEqual({ count: 10, isStale: true }); + }); + + it('should return count >= 0 even if DynamoDB count is negative', async () => { + const now = Date.now(); + mockDynamoDBClient.on(GetItemCommand).resolves({ + Item: { + pk: { S: 'prod#Org#my-org' }, + count: { N: '-5' }, // Negative count due to race conditions + updated: { N: String(now) }, + }, + }); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toEqual({ count: 0, isStale: false }); + }); + + it('should return null on DynamoDB error', async () => { + mockDynamoDBClient.on(GetItemCommand).rejects(new Error('DynamoDB error')); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBeNull(); + }); + + it('should return null when not enabled', async () => { + dynamoDbRunnerCountCache.reset(); + + const result = await dynamoDbRunnerCountCache.get('prod', 'Org', 'my-org'); + expect(result).toBeNull(); + }); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.ts new file mode 100644 index 0000000000..88444838a8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-count-cache.ts @@ -0,0 +1,241 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +const logger = createChildLogger('runner-count-cache'); + +/** + * Cache entry for EC2 runner counts with TTL support. + * This cache helps reduce EC2 DescribeInstances API calls during scale-up operations, + * addressing rate limiting issues in high-volume environments (Issue #4710). + */ +interface EC2RunnerCountCacheEntry { + count: number; + timestamp: number; +} + +/** + * In-memory cache for EC2 runner counts to mitigate EC2 API rate limiting. + * + * This cache stores the count of active runners per environment/type/owner combination + * with a configurable TTL. Within a single Lambda invocation processing batch messages, + * this prevents redundant DescribeInstances calls for the same owner group. + * + * The cache is designed to be slightly stale (short TTL) to reduce API load while + * maintaining accuracy for scaling decisions. In high-throughput environments (20K+ runners/day), + * this can significantly reduce EC2 API throttling issues. + * + * @see https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710 + */ +export class ec2RunnerCountCache { + private static counts: Map = new Map(); + + /** + * Default TTL in milliseconds. 5 seconds provides a good balance between + * reducing API calls and maintaining accuracy for scaling decisions. + */ + private static DEFAULT_TTL_MS = 5000; + + /** + * Resets the cache. Called at the start of each Lambda invocation to ensure + * fresh data for new invocations while still benefiting from caching within + * a single invocation processing multiple messages. + */ + public static reset(): void { + ec2RunnerCountCache.counts.clear(); + } + + /** + * Generates a cache key from the filter parameters. + * Format: "environment#runnerType#runnerOwner" + */ + private static generateKey(environment: string, runnerType: string, runnerOwner: string): string { + return `${environment}#${runnerType}#${runnerOwner}`; + } + + /** + * Gets the cached runner count if available and not expired. + * + * @param environment - The deployment environment (e.g., "prod", "dev") + * @param runnerType - The runner type ("Org" or "Repo") + * @param runnerOwner - The owner (org name or owner/repo) + * @param ttlMs - Optional custom TTL in milliseconds + * @returns The cached count or undefined if not cached or expired + */ + public static get( + environment: string, + runnerType: string, + runnerOwner: string, + ttlMs: number = ec2RunnerCountCache.DEFAULT_TTL_MS, + ): number | undefined { + const key = ec2RunnerCountCache.generateKey(environment, runnerType, runnerOwner); + const cached = ec2RunnerCountCache.counts.get(key); + + if (cached && Date.now() - cached.timestamp < ttlMs) { + return cached.count; + } + + // Entry expired or not found, remove it + if (cached) { + ec2RunnerCountCache.counts.delete(key); + } + + return undefined; + } + + /** + * Sets the runner count in the cache. + * + * @param environment - The deployment environment + * @param runnerType - The runner type + * @param runnerOwner - The owner + * @param count - The current count of runners + */ + public static set(environment: string, runnerType: string, runnerOwner: string, count: number): void { + const key = ec2RunnerCountCache.generateKey(environment, runnerType, runnerOwner); + ec2RunnerCountCache.counts.set(key, { + count, + timestamp: Date.now(), + }); + } + + /** + * Increments the cached count by a specified amount. + * Used after successfully creating new runners to keep the cache accurate + * without requiring a new DescribeInstances call. + * + * @param environment - The deployment environment + * @param runnerType - The runner type + * @param runnerOwner - The owner + * @param increment - The number to add to the current count + */ + public static increment(environment: string, runnerType: string, runnerOwner: string, increment: number): void { + const key = ec2RunnerCountCache.generateKey(environment, runnerType, runnerOwner); + const cached = ec2RunnerCountCache.counts.get(key); + + if (cached) { + cached.count += increment; + cached.timestamp = Date.now(); + } + } + + /** + * Gets the current cache size (for debugging/metrics). + */ + public static size(): number { + return ec2RunnerCountCache.counts.size; + } +} + +/** + * DynamoDB-based persistent cache for EC2 runner counts. + * + * This cache reads from a DynamoDB table that is updated by an EventBridge-triggered + * Lambda function when EC2 instances change state. This provides cross-invocation + * consistency and eliminates EC2 DescribeInstances calls entirely. + * + * The table is expected to have: + * - pk (partition key): "environment#type#owner" format + * - count: atomic counter of active runners + * - updated: timestamp of last update + * + * @see https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710 + */ +export class dynamoDbRunnerCountCache { + private static dynamoClient: DynamoDBClient | null = null; + private static tableName: string | null = null; + private static staleThresholdMs: number = 60000; // 1 minute default + + /** + * Initializes the DynamoDB cache with the required configuration. + * Should be called once at Lambda startup if the cache table is configured. + */ + public static initialize(tableName: string, region: string, staleThresholdMs?: number): void { + dynamoDbRunnerCountCache.tableName = tableName; + dynamoDbRunnerCountCache.dynamoClient = new DynamoDBClient({ region }); + if (staleThresholdMs !== undefined) { + dynamoDbRunnerCountCache.staleThresholdMs = staleThresholdMs; + } + logger.debug('DynamoDB runner count cache initialized', { tableName, staleThresholdMs }); + } + + /** + * Checks if the DynamoDB cache is enabled and initialized. + */ + public static isEnabled(): boolean { + return dynamoDbRunnerCountCache.tableName !== null && dynamoDbRunnerCountCache.dynamoClient !== null; + } + + /** + * Generates a cache key from the filter parameters. + * Format: "environment#runnerType#runnerOwner" + */ + private static generateKey(environment: string, runnerType: string, runnerOwner: string): string { + return `${environment}#${runnerType}#${runnerOwner}`; + } + + /** + * Gets the runner count from DynamoDB if available and not stale. + * + * @param environment - The deployment environment + * @param runnerType - The runner type ("Org" or "Repo") + * @param runnerOwner - The owner (org name or owner/repo) + * @returns Object with count and isStale flag, or null if not found + */ + public static async get( + environment: string, + runnerType: string, + runnerOwner: string, + ): Promise<{ count: number; isStale: boolean } | null> { + if (!dynamoDbRunnerCountCache.isEnabled()) { + return null; + } + + const pk = dynamoDbRunnerCountCache.generateKey(environment, runnerType, runnerOwner); + + try { + const result = await dynamoDbRunnerCountCache.dynamoClient!.send( + new GetItemCommand({ + TableName: dynamoDbRunnerCountCache.tableName!, + Key: { + pk: { S: pk }, + }, + }), + ); + + if (!result.Item) { + logger.debug('No DynamoDB cache entry found', { pk }); + return null; + } + + const count = parseInt(result.Item.count?.N || '0', 10); + const updated = parseInt(result.Item.updated?.N || '0', 10); + const isStale = Date.now() - updated > dynamoDbRunnerCountCache.staleThresholdMs; + + logger.debug('DynamoDB cache hit', { pk, count, isStale, ageMs: Date.now() - updated }); + + // Normalize negative counts to zero. This can happen due to race conditions with + // EventBridge events (e.g., termination event arrives before running event). + if (count < 0) { + logger.warn('DynamoDB cache returned negative count, normalizing to 0', { + pk, + rawCount: count, + updated, + }); + } + + return { count: Math.max(0, count), isStale }; + } catch (error) { + logger.warn('Failed to read from DynamoDB cache', { pk, error }); + return null; + } + } + + /** + * Resets the cache configuration (primarily for testing). + */ + public static reset(): void { + dynamoDbRunnerCountCache.dynamoClient = null; + dynamoDbRunnerCountCache.tableName = null; + dynamoDbRunnerCountCache.staleThresholdMs = 60000; + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up-count-cache.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up-count-cache.test.ts new file mode 100644 index 0000000000..508a03811a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up-count-cache.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./runners', () => ({ + listEC2Runners: vi.fn(), +})); +vi.mock('./runner-config', () => ({ + createRunners: vi.fn(), + loadEc2ProviderConfig: vi.fn().mockReturnValue({}), +})); + +import { listEC2Runners } from './runners'; +import { createRunners } from './runner-config'; +import { createEc2ScaleUpProvider } from './scale-up'; +import { ec2RunnerCountCache, dynamoDbRunnerCountCache } from './runner-count-cache'; + +const mockListRunners = vi.mocked(listEC2Runners); +const mockCreateRunners = vi.mocked(createRunners); +const provider = createEc2ScaleUpProvider(vi.fn()); +const input = { runnerType: 'Repo' as const, runnerOwner: 'acme/app' }; +const cleanEnv = process.env; + +function runners(n: number) { + return Array.from({ length: n }, (_, i) => ({ id: `i-${i}`, type: 'Repo', owner: 'acme/app' })); +} + +beforeEach(() => { + vi.clearAllMocks(); + ec2RunnerCountCache.reset(); + dynamoDbRunnerCountCache.reset(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.ENVIRONMENT = 'prod'; + process.env.RUNNER_COUNT_CACHE_TABLE_NAME = 'runner-counts'; + mockListRunners.mockResolvedValue(runners(7) as never); +}); + +describe('getCurrentRunners with the runner count cache', () => { + it('lists EC2 (DescribeInstances) when the cache table is not configured', async () => { + delete process.env.RUNNER_COUNT_CACHE_TABLE_NAME; + expect(await provider.getCurrentRunners({}, input)).toBe(7); + expect(mockListRunners).toHaveBeenCalledTimes(1); + }); + + it('returns the DynamoDB counter when fresh, without listing EC2', async () => { + vi.spyOn(dynamoDbRunnerCountCache, 'get').mockResolvedValue({ count: 3, isStale: false }); + expect(await provider.getCurrentRunners({}, input)).toBe(3); + expect(mockListRunners).not.toHaveBeenCalled(); + }); + + it('falls back to listing EC2 when the counter is stale', async () => { + vi.spyOn(dynamoDbRunnerCountCache, 'get').mockResolvedValue({ count: 3, isStale: true }); + expect(await provider.getCurrentRunners({}, input)).toBe(7); + expect(mockListRunners).toHaveBeenCalledTimes(1); + }); + + it('falls back to listing EC2 on a counter miss, then serves the in-memory value', async () => { + vi.spyOn(dynamoDbRunnerCountCache, 'get').mockResolvedValue(null); + expect(await provider.getCurrentRunners({}, input)).toBe(7); // lists EC2 + expect(await provider.getCurrentRunners({}, input)).toBe(7); // in-memory + expect(mockListRunners).toHaveBeenCalledTimes(1); + }); + + it('resets the in-memory count after creating runners', async () => { + mockCreateRunners.mockResolvedValue({ + instances: ['i-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + } as never); + ec2RunnerCountCache.set('prod', 'Repo', 'acme/app', 5); + expect(ec2RunnerCountCache.get('prod', 'Repo', 'acme/app')).toBe(5); + await provider.createRunners({ + githubRunnerConfig: {}, + numberOfRunners: 1, + githubInstallationClient: {}, + state: {}, + } as never); + expect(ec2RunnerCountCache.get('prod', 'Repo', 'acme/app')).toBeUndefined(); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts index d72edaf7a4..3c68cb1441 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts @@ -18,6 +18,7 @@ import { } from './dynamic-labels'; import { createRunners, loadEc2ProviderConfig } from './runner-config'; import type { CreateEC2RunnerConfig } from './runner-config'; +import { ec2RunnerCountCache, dynamoDbRunnerCountCache } from './runner-count-cache'; const logger = createChildLogger('ec2-scale-up'); @@ -55,11 +56,73 @@ async function resolveEc2LabelsForRunners(messageLabels: string[]): Promise { - return (await listEC2Runners({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length; + const listActual = async (): Promise => + (await listEC2Runners({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length; + + const tableName = runnerCountCacheTableName(); + if (!tableName) { + return listActual(); // feature disabled -> authoritative EC2 listing (unchanged behaviour) + } + + ensureRunnerCountCacheInitialized(tableName); + const environment = process.env.ENVIRONMENT ?? ''; + + // 1) in-memory (dedupes repeated reads within an invocation) + const memo = ec2RunnerCountCache.get(environment, runnerType, runnerOwner); + if (memo !== undefined) { + return memo; + } + + // 2) DynamoDB counter, if fresh + const cached = await dynamoDbRunnerCountCache.get(environment, runnerType, runnerOwner); + if (cached && !cached.isStale) { + ec2RunnerCountCache.set(environment, runnerType, runnerOwner, cached.count); + return cached.count; + } + if (cached?.isStale) { + logger.debug('Runner count cache stale, falling back to EC2 DescribeInstances', { + environment, + runnerType, + runnerOwner, + }); + } + + // 3) miss or stale -> authoritative EC2 listing, then memoise briefly + const actual = await listActual(); + ec2RunnerCountCache.set(environment, runnerType, runnerOwner, actual); + return actual; } async function createEc2ScaleUpRunners( @@ -68,7 +131,7 @@ async function createEc2ScaleUpRunners( ): Promise { const config = loadEc2ScaleUpProviderConfig(); - return await createRunners( + const result = await createRunners( githubRunnerConfig, { ...config, @@ -79,6 +142,13 @@ async function createEc2ScaleUpRunners( createStartRunnerConfig, 'scale-up-lambda', ); + + // New runners now exist; drop the short-lived in-memory count so a later read + // in the same invocation reflects them via the DescribeInstances fallback. The + // DynamoDB counter converges shortly after via the EventBridge counter Lambda. + ec2RunnerCountCache.reset(); + + return result; } export function createEc2ScaleUpProvider( diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index 9d39fd294a..36258dda88 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -26,6 +26,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", + "@aws-sdk/client-dynamodb": "^3.1009.0", "@aws-sdk/client-ec2": "^3.1009.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 56ae435c2c..c2c7fc6eaa 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -147,6 +147,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-sdk/client-dynamodb": "npm:^3.1009.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" @@ -209,6 +210,22 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/runner-count-cache@workspace:functions/runner-count-cache": + version: 0.0.0-use.local + resolution: "@aws-github-runner/runner-count-cache@workspace:functions/runner-count-cache" + dependencies: + "@aws-github-runner/aws-powertools-util": "npm:*" + "@aws-sdk/client-dynamodb": "npm:^3.948.0" + "@aws-sdk/client-ec2": "npm:^3.948.0" + "@aws-sdk/types": "npm:^3.936.0" + "@types/aws-lambda": "npm:^8.10.155" + "@types/node": "npm:^22.19.0" + "@vercel/ncc": "npm:^0.38.4" + aws-sdk-client-mock: "npm:^4.1.0" + aws-sdk-client-mock-jest: "npm:^4.1.0" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -342,6 +359,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1009.0": + version: 3.1110.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1110.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/credential-provider-node": "npm:^3.972.79" + "@aws-sdk/dynamodb-codec": "npm:^3.973.42" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.28" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/03360588d6f24961f5fabe6799c3a5426ab587b045333f7f4d989417ec51ccc7a14efcee6b46d3dd0e27412cbf6b0d3c48754326436c396de46ce1cd9edf54a3 + languageName: node + linkType: hard + +"@aws-sdk/client-dynamodb@npm:^3.948.0": + version: 3.1108.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1108.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/credential-provider-node": "npm:^3.972.79" + "@aws-sdk/dynamodb-codec": "npm:^3.973.42" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.28" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/f72760e4967709a31b9d3b6615116ce1161aaed05ace32b92c8d916c809ca6a5aed61f9f4d3d3cf2ecbf00ca077bddbc00f6f5df88c7c2dc9392c19479551d66 + languageName: node + linkType: hard + "@aws-sdk/client-ec2@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-ec2@npm:3.1014.0" @@ -391,6 +444,23 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-ec2@npm:^3.948.0": + version: 3.1108.0 + resolution: "@aws-sdk/client-ec2@npm:3.1108.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/credential-provider-node": "npm:^3.972.79" + "@aws-sdk/middleware-sdk-ec2": "npm:^3.972.56" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/7b33f255a983b66517520124b060a0c17cbdcc13a5eb67f4b86db64da9ca4c68de3a27ae982f212e154bdc82c6bd5c30d7e560dc0ce6944f3696fbc61d4ddd28 + languageName: node + linkType: hard + "@aws-sdk/client-eventbridge@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-eventbridge@npm:3.1014.0" @@ -620,6 +690,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.7": + version: 3.977.7 + resolution: "@aws-sdk/core@npm:3.977.7" + dependencies: + "@aws-sdk/types": "npm:^3.974.3" + "@aws-sdk/xml-builder": "npm:^3.972.38" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/305bc5d7bd61b33bbdbec7abc5dbf03fb64ea8e2b60fcda9e4ab22ce2fc09eea6db71769010ead7437e75d09c6407acedfe1f8a52e9b1ea97ba5c0ebd3e348e0 + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -643,6 +729,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.68": + version: 3.972.68 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.68" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/31b9d5fd71d00556ecf1bc5da793e8cb04f98d09ccb00c8b65a59b88a6a860a7737e1aeefcb5f7e62f322f9f3a7eba279b205c8623e912d936fe01a667ab724a + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -661,6 +760,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.70": + version: 3.972.70 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.70" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/9230d722c0307bbafd0b56a42c25708a25a0d087c9993c37f628fbae8e142120d73a34738b6b0bf15f57efdbcf664feb6ae773e2526b3729663ce54403393487 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -683,6 +797,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.13": + version: 3.973.13 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.13" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/credential-provider-env": "npm:^3.972.68" + "@aws-sdk/credential-provider-http": "npm:^3.972.70" + "@aws-sdk/credential-provider-login": "npm:^3.972.75" + "@aws-sdk/credential-provider-process": "npm:^3.972.68" + "@aws-sdk/credential-provider-sso": "npm:^3.973.12" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.74" + "@aws-sdk/nested-clients": "npm:^3.997.42" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0d7c76a3227f21e8c08af3db03e369651c9efb795b964707f243d69bdc1fb322b712a1be7e7d474ffaca25118eb00ce6eeb56a646850d72a14a8f7e6120cc02c + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -699,6 +834,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.75": + version: 3.972.75 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.75" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/nested-clients": "npm:^3.997.42" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/ab05ddced3fe6bc9360b79950fb07ed01a06c103771c36686d3495135a18d55ebbc8b5fb56744b120e2cc2cc9e413615ce6e1b1c5186ca97b8eb68142162081e + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -719,6 +868,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.79": + version: 3.972.79 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.79" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.68" + "@aws-sdk/credential-provider-http": "npm:^3.972.70" + "@aws-sdk/credential-provider-ini": "npm:^3.973.13" + "@aws-sdk/credential-provider-process": "npm:^3.972.68" + "@aws-sdk/credential-provider-sso": "npm:^3.973.12" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.74" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0c7b282b9b8a6774e0acc987e1fff59d10926afe336b0dfe0fa4af84d333b46df50f8679b2cce95d73b75b0ae593b44ad4896bc8d54e337a3b611666cf1ceb6a + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -733,6 +901,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.68": + version: 3.972.68 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.68" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/97a7061fdf997588ab8ea2feb4a5a268e3c94e841e3418faf683fade3bca1c627b6e30969666d7ca96207a37fe7178e704eb8adc720ac8eb391e8c1e85291f11 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -749,6 +930,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/nested-clients": "npm:^3.997.42" + "@aws-sdk/token-providers": "npm:3.1108.0" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/061f0219ace75565abe79480f81920eb4bfe45e84b53901941c3b7d6c9aee7a67542e1058fbeb40139fd4887bb42cdf759ba1ebdb7b7cbb2fa9cd1e033e32871 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -764,6 +960,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/nested-clients": "npm:^3.997.42" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/9c1da6479fdc329a7540420e0e5a1030f159f689150acb7d1d67ae9c91b99415b47f0abce859bc8987e9a7aee109e411e45f0e197b5e1d94c3c31379c8a426b2 + languageName: node + linkType: hard + +"@aws-sdk/dynamodb-codec@npm:^3.973.42": + version: 3.973.42 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.42" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/9e4489b2d90112aab0e98dde04d2e787f31ab1e06e4473ce85f7eff0851e9461720707fd2cd5ae6d48e70635b30c979066ea6bd3e396aaad37ad39e494be9690 + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.10": + version: 3.972.10 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.10" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/cdcad9be7fe0c6ace273ddeaa145f397653d2866fff5a3620410d389f45b84bf2856269c9c404f082d52d1a9849b83608f809582b7689cf35abd4affdf7af97d + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -796,6 +1028,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.28": + version: 3.972.28 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.28" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.10" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6bcbc9b3f28664d075997ed4b1fe3a1725e45f9d7c00265d08d022cb0acc7a00519cc5e922d50f8e9fcd1eb5d2158a888d46a6c2c6e78a7832cf54d0c902b478 + languageName: node + linkType: hard + "@aws-sdk/middleware-expect-continue@npm:^3.972.8": version: 3.972.8 resolution: "@aws-sdk/middleware-expect-continue@npm:3.972.8" @@ -893,6 +1138,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-sdk-ec2@npm:^3.972.56": + version: 3.972.56 + resolution: "@aws-sdk/middleware-sdk-ec2@npm:3.972.56" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/8f01970747059219f7d4cdc1aea27adbb401b95970042094db0f011394bc581bcc0c984a3ce2dc8116aed8a47982d86fb63c094ec38764181650984e60170d28 + languageName: node + linkType: hard + "@aws-sdk/middleware-sdk-s3@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/middleware-sdk-s3@npm:3.972.23" @@ -1002,6 +1261,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.42": + version: 3.997.42 + resolution: "@aws-sdk/nested-clients@npm:3.997.42" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.44" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1405a73a86aa904fa503592fcc0793dc5b8ab994d1c9eda1b017ca26bd1a659fd87cd9af762c713466ebc8950ced06ee49521a5dea25a99edbe011720e1d86e3 + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1029,6 +1304,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.44": + version: 3.996.44 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.44" + dependencies: + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b9c970abad58f9f87dcb2577f9121c7c9015ea18b4101e4813ff92bb5c19524c8634f6507f3c273ad8694d5ef6f272d5bd0648993c16431fcb09dcc402dd7d72 + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1044,6 +1331,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1108.0": + version: 3.1108.0 + resolution: "@aws-sdk/token-providers@npm:3.1108.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.7" + "@aws-sdk/nested-clients": "npm:^3.997.42" + "@aws-sdk/types": "npm:^3.974.3" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/4593b1228d55cfc960cb22c6eecfe29032f4b1a01f0452f46ba22cdaa7bcde051f5749cfcc465ef682f659cc7a4f0d7c55ce254158b88eacf3ca1c3fa7946e7e + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1054,6 +1355,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.936.0, @aws-sdk/types@npm:^3.974.3": + version: 3.974.3 + resolution: "@aws-sdk/types@npm:3.974.3" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6850403d8d9358ea497a63eaa50129f1989a2acc959878bb473525f04238099ab85f9bb7877f9d03191f3397941cf841ce80aaf0f319c94273e41fedf51661d7 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1139,6 +1450,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.38": + version: 3.972.38 + resolution: "@aws-sdk/xml-builder@npm:3.972.38" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/8f4c500bea2e1060b2cf47aac4be87b80e9bdf5389d3d3aeaca66397ebc4b15cdc39a8d3f0d54df2c39597c3c2957edce4cca0d92a7a875aedcfdb094d6910ec + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1146,6 +1467,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.23.5, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4404,6 +4732,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1, @smithy/core@npm:^3.32.0": + version: 3.32.0 + resolution: "@smithy/core@npm:3.32.0" + dependencies: + "@smithy/types": "npm:^4.17.0" + tslib: "npm:^2.6.2" + checksum: 10c0/2d3a973587715641bbe3f46b09337024f181d20cc0baddb402c42eb9aecd17bc92fbcc61f0b248ddcc75982266656fdeb3ca100509d6a3a412e6c54c5f61722e + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4417,6 +4755,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.5.0 + resolution: "@smithy/credential-provider-imds@npm:4.5.0" + dependencies: + "@smithy/core": "npm:^3.32.0" + "@smithy/types": "npm:^4.17.0" + tslib: "npm:^2.6.2" + checksum: 10c0/b639b737fe6a03f5c0bc8e95301a041f9ec3df9d2adc55e59a8079e73da0e8865511bb67f9f052e492a3413a84acd02b6ba1b34972e886c3311ee3bd77afbfdb + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4485,6 +4834,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.7.0 + resolution: "@smithy/fetch-http-handler@npm:5.7.0" + dependencies: + "@smithy/core": "npm:^3.32.0" + "@smithy/types": "npm:^4.17.0" + tslib: "npm:^2.6.2" + checksum: 10c0/2384d4f000855c8f1b097f959136820da1f133bf5c73883d2365c295776b0142f5bc3660721ee66e99d2c6a1ccce93158ee2c42f5983cf9f00d86ccab8c5dd0b + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4650,6 +5010,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.10.0 + resolution: "@smithy/node-http-handler@npm:4.10.0" + dependencies: + "@smithy/core": "npm:^3.32.0" + "@smithy/types": "npm:^4.17.0" + tslib: "npm:^2.6.2" + checksum: 10c0/6cf7e09b943c6b291aa7abaf5db1711643a1b25fe36eacc8fbfd2225616be26b22e73a42dd90a9b953836bf1503ae9a2d24d601bf1d97d49b00e582f1dd81a7d + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4735,6 +5106,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.7.0 + resolution: "@smithy/signature-v4@npm:5.7.0" + dependencies: + "@smithy/core": "npm:^3.32.0" + "@smithy/types": "npm:^4.17.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4ac74ae7f1e6f40ca153ae2adb185754871c8696bb176a0b759e275b1bc634b4d82f2e85d6bad12a1ec8efb0f26c424328f9e39ce58a5ac6c2ba7a72ed992128 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4768,6 +5150,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1, @smithy/types@npm:^4.17.0": + version: 4.17.0 + resolution: "@smithy/types@npm:4.17.0" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/f985f116e02ad60168a4bcd97140e971ee0a83a083574a8800d3364c62c2445d1fa2314214f19d16446acda18bed9f0ee632cacdd897804c51339a5a1c9ce422 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" @@ -5233,6 +5624,13 @@ __metadata: languageName: node linkType: hard +"@types/aws-lambda@npm:^8.10.155": + version: 8.10.162 + resolution: "@types/aws-lambda@npm:8.10.162" + checksum: 10c0/0b93ebe339bf79d40e0811fb6ba658b425a5f9a45c2b07a2240e4260036ac949e6e5a5776db6269bc71218f127b046d7793f950f61ed7e94262b4fecf643eb7c + languageName: node + linkType: hard + "@types/aws-lambda@npm:^8.10.159": version: 8.10.159 resolution: "@types/aws-lambda@npm:8.10.159" @@ -5385,6 +5783,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^22.19.0": + version: 22.20.1 + resolution: "@types/node@npm:22.20.1" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/f2ba54d3d1fb92e1c57c78d32c3a17655b1e87363b707136f55c422b4838d4054901ce5d27f75bb0e5ecb7ebfee3804e0987822d22b473473008091857a09353 + languageName: node + linkType: hard + "@types/parse-json@npm:^4.0.0": version: 4.0.2 resolution: "@types/parse-json@npm:4.0.2" @@ -8886,6 +9293,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "moment-timezone@npm:^0.6.0": version: 0.6.0 resolution: "moment-timezone@npm:0.6.0" @@ -9288,6 +9704,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "obug@npm:^2.1.1": version: 2.1.1 resolution: "obug@npm:2.1.1" diff --git a/main.tf b/main.tf index cad9b66c58..79aee4e8b3 100644 --- a/main.tf +++ b/main.tf @@ -231,6 +231,11 @@ module "runners" { license_specifications = var.runner_license_specifications use_dedicated_host = var.use_dedicated_host + runner_count_cache = var.runner_count_cache.enable ? { + table_name = module.runner_count_cache[0].dynamodb_table.name + stale_threshold_ms = var.runner_count_cache.stale_threshold_ms + } : null + enable_runner_binaries_syncer = var.enable_runner_binaries_syncer lambda_s3_bucket = var.lambda_s3_bucket runners_lambda_s3_key = var.runners_lambda_s3_key @@ -419,3 +424,24 @@ module "instance_termination_watcher" { config = merge(local.lambda_instance_termination_watcher, var.instance_termination_watcher) } + +module "runner_count_cache" { + source = "./modules/runner-count-cache" + count = var.runner_count_cache.enable ? 1 : 0 + + prefix = var.prefix + tags = local.tags + environment_filter = var.prefix + + lambda_runtime = var.lambda_runtime + lambda_architecture = var.lambda_architecture + lambda_s3_bucket = var.lambda_s3_bucket + + counter_lambda_s3_key = var.runner_count_cache.lambda_s3_key + counter_lambda_s3_object_version = var.runner_count_cache.lambda_s3_object_version + counter_lambda_memory_size = var.runner_count_cache.lambda_memory_size + counter_lambda_timeout = var.runner_count_cache.lambda_timeout + + ttl_seconds = var.runner_count_cache.ttl_seconds + cache_stale_threshold_ms = var.runner_count_cache.stale_threshold_ms +} diff --git a/modules/runner-count-cache/README.md b/modules/runner-count-cache/README.md new file mode 100644 index 0000000000..96038cc981 --- /dev/null +++ b/modules/runner-count-cache/README.md @@ -0,0 +1,98 @@ +# Runner Count Cache Module + +This module provides a DynamoDB-based caching system for tracking the number of active EC2 runners. It significantly reduces the need for EC2 `DescribeInstances` API calls during scale-up operations, addressing performance bottlenecks in high-volume environments. + +## Problem Statement + +In large-scale deployments (20K+ runners per day), the scale-up Lambda function's use of `DescribeInstances` to count current runners can: + +- Cause EC2 API rate limiting (throttling) +- Add 15+ seconds of latency to scaling decisions +- Impact overall scaling performance + +See [Issue #4710](https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710) for details. + +## Solution Architecture + +``` +┌─────────────────┐ ┌─────────────────┐ +│ EC2 Instance │ State Change │ EventBridge │ +│ Lifecycle │ ─────────────────► │ Rule │ +└─────────────────┘ └────────┬────────┘ + │ + ▼ + ┌────────────────┐ + │ Counter Lambda │ + │ (update count) │ + └───────┬────────┘ + │ + ▼ +┌─────────────────┐ ┌───────────────────┐ +│ Scale-Up Lambda │ ◄──── Read ─────── │ DynamoDB Table │ +│ (check limit) │ │ ┌───────────────┐ │ +└─────────────────┘ │ │ pk: env#type │ │ + │ │ │ count: 42 │ │ + │ Fallback if stale │ │ updated: ts │ │ + ▼ │ └───────────────┘ │ +┌─────────────────┐ └───────────────────┘ +│ EC2 Describe │ +│ Instances │ +└─────────────────┘ +``` + +## Features + +- **Event-driven**: Uses EventBridge to react to EC2 state changes in real-time +- **Atomic counters**: DynamoDB atomic increments/decrements prevent race conditions +- **Auto-cleanup**: TTL on DynamoDB items prevents stale data accumulation +- **Fallback support**: Scale-up Lambda falls back to EC2 API if cache is stale +- **Low cost**: PAY_PER_REQUEST billing, typically pennies per month + +## Usage + +```hcl +module "runner_count_cache" { + source = "./modules/runner-count-cache" + + prefix = "github-runners" + environment_filter = "production" + + tags = { + Environment = "production" + } +} +``` + +## Integration with Scale-Up Lambda + +The scale-up Lambda can be configured to use this cache by setting these environment variables: + +- `RUNNER_COUNT_CACHE_TABLE_NAME`: DynamoDB table name +- `RUNNER_COUNT_CACHE_STALE_THRESHOLD_MS`: Maximum age of cached counts (default: 60000) + +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 1.3.0 | +| aws | >= 6.21 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| prefix | The prefix used for naming resources | `string` | n/a | yes | +| environment_filter | The environment tag value to filter EC2 instances | `string` | n/a | yes | +| tags | Map of tags to add to resources | `map(string)` | `{}` | no | +| kms_key_arn | Optional CMK Key ARN for DynamoDB encryption | `string` | `null` | no | +| cache_stale_threshold_ms | Max age before cache is considered stale | `number` | `60000` | no | +| ttl_seconds | TTL for DynamoDB items in seconds | `number` | `86400` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| dynamodb_table | DynamoDB table name and ARN | +| lambda_function | Counter Lambda function name and ARN | +| eventbridge_rule | EventBridge rule name and ARN | +| cache_config | Configuration for scale-up Lambda | diff --git a/modules/runner-count-cache/lambda.tf b/modules/runner-count-cache/lambda.tf new file mode 100644 index 0000000000..b9d76da55c --- /dev/null +++ b/modules/runner-count-cache/lambda.tf @@ -0,0 +1,152 @@ +# Counter Lambda Function +# Updates DynamoDB counter when EC2 instances change state + +data "aws_region" "current" {} +data "aws_caller_identity" "current" {} + +locals { + lambda_zip = "${path.module}/../../lambdas/functions/runner-count-cache/runner-count-cache.zip" +} + +resource "aws_lambda_function" "counter" { + s3_bucket = var.lambda_s3_bucket != null ? var.lambda_s3_bucket : null + s3_key = var.counter_lambda_s3_key != null ? var.counter_lambda_s3_key : null + s3_object_version = var.counter_lambda_s3_object_version != null ? var.counter_lambda_s3_object_version : null + filename = var.lambda_s3_bucket == null ? local.lambda_zip : null + source_code_hash = var.lambda_s3_bucket == null && fileexists(local.lambda_zip) ? filebase64sha256(local.lambda_zip) : null + + function_name = "${var.prefix}-runner-count-cache" + role = aws_iam_role.counter.arn + handler = "index.handler" + runtime = var.lambda_runtime + timeout = var.counter_lambda_timeout + memory_size = var.counter_lambda_memory_size + architectures = [var.lambda_architecture] + tags = merge(local.tags, var.lambda_tags) + + environment { + variables = { + DYNAMODB_TABLE_NAME = aws_dynamodb_table.runner_counts.name + ENVIRONMENT_FILTER = var.environment_filter + TTL_SECONDS = var.ttl_seconds + LOG_LEVEL = "info" + POWERTOOLS_SERVICE_NAME = "runner-count-cache" + } + } + + dynamic "vpc_config" { + for_each = var.lambda_subnet_ids != null && var.lambda_security_group_ids != null ? [true] : [] + content { + security_group_ids = var.lambda_security_group_ids + subnet_ids = var.lambda_subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.tracing_config.mode != null ? [true] : [] + content { + mode = var.tracing_config.mode + } + } +} + +resource "aws_cloudwatch_log_group" "counter" { + name = "/aws/lambda/${aws_lambda_function.counter.function_name}" + retention_in_days = var.logging_retention_in_days + kms_key_id = var.logging_kms_key_id + tags = local.tags +} + +# IAM Role for Counter Lambda +resource "aws_iam_role" "counter" { + name = "${var.prefix}-runner-count-cache" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.role_path + permissions_boundary = var.role_permissions_boundary + tags = local.tags +} + +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +# Policy for DynamoDB access +resource "aws_iam_role_policy" "counter_dynamodb" { + name = "dynamodb-access" + role = aws_iam_role.counter.id + policy = data.aws_iam_policy_document.counter_dynamodb.json +} + +data "aws_iam_policy_document" "counter_dynamodb" { + statement { + sid = "DynamoDBAccess" + actions = [ + "dynamodb:UpdateItem", + "dynamodb:GetItem", + "dynamodb:PutItem", + ] + resources = [aws_dynamodb_table.runner_counts.arn] + } +} + +# Policy for EC2 DescribeInstances (to get instance tags) +resource "aws_iam_role_policy" "counter_ec2" { + name = "ec2-describe" + role = aws_iam_role.counter.id + policy = data.aws_iam_policy_document.counter_ec2.json +} + +data "aws_iam_policy_document" "counter_ec2" { + statement { + sid = "EC2DescribeInstances" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + ] + # EC2 Describe* actions require resource = "*" - they cannot be scoped to specific + # instance ARNs. See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-policy-structure.html + # The Lambda filters instances by tags after fetching to ensure only runner instances are counted. + resources = ["*"] + } +} + +# Policy for CloudWatch Logs +resource "aws_iam_role_policy" "counter_logs" { + name = "cloudwatch-logs" + role = aws_iam_role.counter.id + policy = data.aws_iam_policy_document.counter_logs.json +} + +data "aws_iam_policy_document" "counter_logs" { + statement { + sid = "CloudWatchLogs" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "${aws_cloudwatch_log_group.counter.arn}:*", + ] + } +} + +# VPC policy if Lambda is in VPC +resource "aws_iam_role_policy_attachment" "counter_vpc" { + count = var.lambda_subnet_ids != null ? 1 : 0 + role = aws_iam_role.counter.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +# X-Ray tracing policy +resource "aws_iam_role_policy_attachment" "counter_xray" { + count = var.tracing_config.mode != null ? 1 : 0 + role = aws_iam_role.counter.name + policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess" +} diff --git a/modules/runner-count-cache/main.tf b/modules/runner-count-cache/main.tf new file mode 100644 index 0000000000..a9ac1c241d --- /dev/null +++ b/modules/runner-count-cache/main.tf @@ -0,0 +1,137 @@ +# Runner Count Cache Module +# +# This module creates a DynamoDB-based cache for tracking the number of active +# EC2 runners. It uses EventBridge to listen for EC2 state changes and updates +# a counter in DynamoDB, significantly reducing the need for DescribeInstances +# API calls during scale-up operations. +# +# This addresses the performance bottleneck described in Issue #4710: +# https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710 + +locals { + tags = var.tags +} + +# DynamoDB table to store runner counts per environment/type/owner +resource "aws_dynamodb_table" "runner_counts" { + name = "${var.prefix}-runner-counts" + billing_mode = "PAY_PER_REQUEST" # Auto-scales with no provisioning needed + + hash_key = "pk" # Format: "environment#runnerType#runnerOwner" + + attribute { + name = "pk" + type = "S" + } + + ttl { + attribute_name = "ttl" + enabled = true + } + + # Optional encryption with customer-managed KMS key + dynamic "server_side_encryption" { + for_each = var.kms_key_arn != null ? [1] : [] + content { + enabled = true + kms_key_arn = var.kms_key_arn + } + } + + point_in_time_recovery { + enabled = false # Not needed for cache data + } + + tags = merge(local.tags, { + Name = "${var.prefix}-runner-counts" + }) +} + +# EventBridge rule to capture EC2 instance state changes +resource "aws_cloudwatch_event_rule" "ec2_state_change" { + name = "${var.prefix}-runner-state-change" + description = "Captures EC2 instance state changes for GitHub Action runners" + + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + detail = { + state = ["running", "pending", "terminated", "stopped", "shutting-down"] + } + }) + + tags = local.tags +} + +# EventBridge target to invoke the counter Lambda +resource "aws_cloudwatch_event_target" "counter_lambda" { + rule = aws_cloudwatch_event_rule.ec2_state_change.name + arn = aws_lambda_function.counter.arn + + # EventBridge delivery is at-least-once but not guaranteed. If the counter + # Lambda fails through the whole retry window (e.g. a sustained DynamoDB + # error), the event is dead-lettered rather than dropped silently, which would + # otherwise drift the counter until the read-side staleness fallback corrects it. + retry_policy { + maximum_event_age_in_seconds = 3600 # 1h; older events are reconciled against the provider + maximum_retry_attempts = 10 + } + + dead_letter_config { + arn = aws_sqs_queue.counter_dlq.arn + } +} + +# Dead-letter queue for state-change events that fail delivery to the counter Lambda. +# Captures the silent-loss failure mode (retry exhaustion) for investigation/replay. +resource "aws_sqs_queue" "counter_dlq" { + name = "${var.prefix}-runner-count-dlq" + message_retention_seconds = 1209600 # 14 days + sqs_managed_sse_enabled = true + tags = local.tags +} + +# Allow the state-change rule to write failed deliveries to the DLQ. +resource "aws_sqs_queue_policy" "counter_dlq" { + queue_url = aws_sqs_queue.counter_dlq.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "events.amazonaws.com" } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.counter_dlq.arn + Condition = { + ArnEquals = { "aws:SourceArn" = aws_cloudwatch_event_rule.ec2_state_change.arn } + } + }] + }) +} + +# Alarm when the DLQ is non-empty: state-change events failed all delivery +# attempts, so the counter may be drifting and the messages need replay. +resource "aws_cloudwatch_metric_alarm" "counter_dlq_not_empty" { + alarm_name = "${var.prefix}-runner-count-dlq-not-empty" + alarm_description = "EC2 state-change events failed delivery to the runner count Lambda and landed in the DLQ." + namespace = "AWS/SQS" + metric_name = "ApproximateNumberOfMessagesVisible" + statistic = "Maximum" + period = 300 + evaluation_periods = 1 + threshold = 0 + comparison_operator = "GreaterThanThreshold" + treat_missing_data = "notBreaching" + dimensions = { + QueueName = aws_sqs_queue.counter_dlq.name + } + tags = local.tags +} + +# Permission for EventBridge to invoke the Lambda +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.counter.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.ec2_state_change.arn +} diff --git a/modules/runner-count-cache/outputs.tf b/modules/runner-count-cache/outputs.tf new file mode 100644 index 0000000000..7f9bed01d9 --- /dev/null +++ b/modules/runner-count-cache/outputs.tf @@ -0,0 +1,39 @@ +output "dynamodb_table" { + description = "DynamoDB table for runner counts" + value = { + name = aws_dynamodb_table.runner_counts.name + arn = aws_dynamodb_table.runner_counts.arn + } +} + +output "lambda_function" { + description = "Counter Lambda function" + value = { + name = aws_lambda_function.counter.function_name + arn = aws_lambda_function.counter.arn + } +} + +output "eventbridge_rule" { + description = "EventBridge rule for EC2 state changes" + value = { + name = aws_cloudwatch_event_rule.ec2_state_change.name + arn = aws_cloudwatch_event_rule.ec2_state_change.arn + } +} + +output "lambda_role" { + description = "IAM role for the counter Lambda" + value = { + name = aws_iam_role.counter.name + arn = aws_iam_role.counter.arn + } +} + +output "cache_config" { + description = "Configuration for scale-up Lambda to use the cache" + value = { + table_name = aws_dynamodb_table.runner_counts.name + stale_threshold_ms = var.cache_stale_threshold_ms + } +} diff --git a/modules/runner-count-cache/variables.tf b/modules/runner-count-cache/variables.tf new file mode 100644 index 0000000000..fd5ccfff6a --- /dev/null +++ b/modules/runner-count-cache/variables.tf @@ -0,0 +1,127 @@ +variable "prefix" { + description = "The prefix used for naming resources" + type = string +} + +variable "tags" { + description = "Map of tags that will be added to created resources" + type = map(string) + default = {} +} + +variable "kms_key_arn" { + description = "Optional CMK Key ARN to be used for DynamoDB encryption. If not provided, AWS managed key will be used." + type = string + default = null +} + +variable "environment_filter" { + description = "The environment tag value to filter EC2 instances. Should match the 'ghr:environment' tag value." + type = string +} + +variable "counter_lambda_timeout" { + description = "Timeout for the counter update lambda in seconds." + type = number + default = 30 +} + +variable "counter_lambda_memory_size" { + description = "Memory size limit in MB for counter update lambda." + type = number + default = 256 +} + +variable "lambda_runtime" { + description = "AWS Lambda runtime for the counter function." + type = string + default = "nodejs20.x" +} + +variable "lambda_architecture" { + description = "AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance." + type = string + default = "arm64" +} + +variable "lambda_s3_bucket" { + description = "S3 bucket from which to get the lambda function. When not set, the lambda will be built locally." + type = string + default = null +} + +variable "counter_lambda_s3_key" { + description = "S3 key for the counter lambda function." + type = string + default = null +} + +variable "counter_lambda_s3_object_version" { + description = "S3 object version for the counter lambda function." + type = string + default = null +} + +variable "logging_retention_in_days" { + description = "Specifies the number of days you want to retain log events." + type = number + default = 7 +} + +variable "logging_kms_key_id" { + description = "The KMS Key ARN to use for CloudWatch log group encryption." + type = string + default = null +} + +variable "tracing_config" { + description = "Configuration for lambda tracing." + type = object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }) + default = {} +} + +variable "lambda_subnet_ids" { + description = "List of subnets in which the lambda will be launched." + type = list(string) + default = null +} + +variable "lambda_security_group_ids" { + description = "List of security group IDs associated with the Lambda function." + type = list(string) + default = null +} + +variable "role_permissions_boundary" { + description = "Permissions boundary that will be added to the created role for the lambda." + type = string + default = null +} + +variable "role_path" { + description = "The path that will be added to the role." + type = string + default = null +} + +variable "lambda_tags" { + description = "Map of tags to add to the Lambda function." + type = map(string) + default = {} +} + +variable "ttl_seconds" { + description = "TTL for DynamoDB items in seconds. Items older than this will be automatically deleted." + type = number + default = 86400 # 24 hours +} + +variable "cache_stale_threshold_ms" { + description = "Maximum age in milliseconds before a cached count is considered stale and falls back to EC2 API." + type = number + default = 60000 # 60 seconds +} diff --git a/modules/runner-count-cache/versions.tf b/modules/runner-count-cache/versions.tf new file mode 100644 index 0000000000..42a40b33fd --- /dev/null +++ b/modules/runner-count-cache/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.21" + } + } +} diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 2e045345a6..3489b55feb 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -66,6 +66,8 @@ resource "aws_lambda_function" "scale_up" { SCALE_ERRORS = jsonencode(var.scale_errors) JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) USE_DEDICATED_HOST = var.use_dedicated_host + RUNNER_COUNT_CACHE_TABLE_NAME = try(var.runner_count_cache.table_name, "") + RUNNER_COUNT_CACHE_STALE_THRESHOLD_MS = try(var.runner_count_cache.stale_threshold_ms, 60000) } } @@ -137,6 +139,20 @@ resource "aws_iam_role_policy" "scale_up" { }) } +resource "aws_iam_role_policy" "scale_up_runner_count_cache" { + count = var.runner_count_cache != null ? 1 : 0 + name = "runner-count-cache-policy" + role = aws_iam_role.scale_up.name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = "arn:${var.aws_partition}:dynamodb:${var.aws_region}:${data.aws_caller_identity.current.account_id}:table/${var.runner_count_cache.table_name}" + }] + }) +} + resource "aws_iam_role_policy" "scale_up_logging" { name = "logging-policy" role = aws_iam_role.scale_up.name diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 946f9abf30..58fe3b998c 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -889,3 +889,19 @@ variable "use_dedicated_host" { type = bool default = false } + +variable "runner_count_cache" { + description = <<-EOF + Configuration for the runner count cache feature that reduces the compute provider's + runner listing calls (e.g. EC2 DescribeInstances) during scale-up. Passed from the root + module when the feature is enabled; null disables it (scale-up reads the count directly). + + `table_name`: DynamoDB table holding the per-(environment#type#owner) counter. + `stale_threshold_ms`: age after which a cached count is treated as stale and re-read. + EOF + type = object({ + table_name = string + stale_threshold_ms = number + }) + default = null +} diff --git a/variables.runner-count-cache.tf b/variables.runner-count-cache.tf new file mode 100644 index 0000000000..3065f322b2 --- /dev/null +++ b/variables.runner-count-cache.tf @@ -0,0 +1,27 @@ +variable "runner_count_cache" { + description = <<-EOF + Configuration for the runner count cache feature. Reduces the compute provider's runner + listing calls (e.g. EC2 DescribeInstances) during scale-up by maintaining an event-driven + count of active runners in DynamoDB, addressing API rate limiting in high-volume + environments. See https://github.com/github-aws-runners/terraform-aws-github-runner/issues/4710 + + `enable`: Enable or disable the runner count cache feature. + `stale_threshold_ms`: Age (ms) after which a cached count is considered stale and scale-up falls back to the provider. Default 60000. + `ttl_seconds`: TTL for DynamoDB items in seconds. Default 86400. + `lambda_memory_size`: Memory (MB) of the counter lambda. + `lambda_timeout`: Timeout (seconds) of the counter lambda. + `lambda_s3_key`: S3 key for the lambda artifact. Required if lambdas are sourced from S3. + `lambda_s3_object_version`: S3 object version for the lambda artifact. + EOF + + type = object({ + enable = optional(bool, false) + stale_threshold_ms = optional(number, 60000) + ttl_seconds = optional(number, 86400) + lambda_memory_size = optional(number, 256) + lambda_timeout = optional(number, 30) + lambda_s3_key = optional(string, null) + lambda_s3_object_version = optional(string, null) + }) + default = {} +}