From e737293540562ecb641e67d9c6781b5e765346c5 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 19:44:10 +0200 Subject: [PATCH 01/21] refactor(compute-providers): isolate EC2 provider handling --- .../control-plane/src/pool/pool.test.ts | 4 +- .../src/scale-runners/scale-up.test.ts | 6 +- .../src/runners/aws-dynamic-labels-policy.ts | 1 - .../webhook/src/runners/aws-dynamic-labels.ts | 29 ----- .../webhook/src/runners/dispatch.test.ts | 113 ++++++------------ .../functions/webhook/src/runners/dispatch.ts | 4 +- .../aws/dynamic-labels-policy.ts | 61 ++++++++++ .../ec2/src/webhook/dynamic-labels-policy.ts | 54 +-------- .../ec2/src/webhook/dynamic-labels.test.ts | 58 +++++++++ .../compute-providers/provider-types.test.ts | 11 +- .../compute-providers/webhook.test.ts} | 16 +-- lambdas/libs/compute-providers/webhook.ts | 28 ++++- 12 files changed, 205 insertions(+), 180 deletions(-) delete mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts delete mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts create mode 100644 lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts rename lambdas/{functions/webhook/src/runners/aws-dynamic-labels.test.ts => libs/compute-providers/webhook.test.ts} (72%) diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 568403c3be..ee41d77b41 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -247,8 +247,8 @@ describe('Test simple pool.', () => { }); it('Rejects unsupported pool provider types.', async () => { - await expect(adjust({ poolSize: 10, type: 'microvm' })).rejects.toThrow( - "Unsupported compute provider type 'microvm'", + await expect(adjust({ poolSize: 10, type: 'unsupported-provider' })).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", ); expect(mockListRunners).not.toHaveBeenCalled(); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index eb6899ff79..9df79ceac1 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -2157,9 +2157,11 @@ describe('compute provider selection', () => { }); it('rejects unsupported scale-up provider types', async () => { - process.env.COMPUTE_PROVIDER_TYPE = 'microvm'; + process.env.COMPUTE_PROVIDER_TYPE = 'unsupported-provider'; - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow("Unsupported compute provider type 'microvm'"); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", + ); expect(mockedAppAuth).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts deleted file mode 100644 index 98bba55b30..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts +++ /dev/null @@ -1 +0,0 @@ -export type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '@aws-github-runner/compute-providers'; diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts deleted file mode 100644 index 418697398a..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import type { DynamicLabelDispatchTarget } from '@aws-github-runner/compute-providers'; -import { normalizeComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { webhookProviderRegistry } from '@aws-github-runner/compute-providers/webhook'; - -import type { RunnerMatcherConfig } from '../sqs'; - -const logger = createChildLogger('handler'); - -export function selectAwsDynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - const provider = normalizeComputeProviderType(queue.computeProvider); - const dynamicLabels = provider ? webhookProviderRegistry.capability(provider, 'dynamicLabels') : undefined; - - if (!dynamicLabels) { - logger.warn(`Queue ${queue.id} has unsupported compute provider '${provider ?? String(queue.computeProvider)}'`); - continue; - } - - const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); - if (target) return target; - } - - return undefined; -} diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index ae571da9d8..bb2cdc7cce 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,4 +1,5 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -14,6 +15,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; vi.mock('../sqs'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/compute-providers/webhook', () => ({ + selectDynamicLabelQueue: vi.fn(), +})); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; @@ -246,7 +250,14 @@ describe('Dispatcher', () => { describe('per-matcher dynamic labels handling', () => { const baseRunner = runnerConfig[0]; - it('strips invalid ghr- labels (too long, bad chars) before policy and dispatch', async () => { + beforeEach(() => { + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches, nonGhrLabels, sanitizedGhrLabels) => ({ + queue: matches[0], + labels: [...nonGhrLabels, ...sanitizedGhrLabels], + })); + }); + + it('strips invalid ghr- labels before provider selection and dispatch', async () => { const longLabel = 'ghr-' + 'a'.repeat(125); // 129 chars config = await createConfig(undefined, [ { @@ -276,19 +287,25 @@ describe('Dispatcher', () => { } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); + expect(selectDynamicLabelQueue).toHaveBeenCalledWith( + [expect.objectContaining({ id: baseRunner.id })], + ['self-hosted', 'linux'], + ['ghr-valid:value', 'ghr-list:value;another'], + ); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-valid:value', 'ghr-list:value;another'] }), ); }); - it('rejects the job (202) when the only matching runner has enableDynamicLabels=false', async () => { + it('rejects the job when no provider accepts the dynamic labels', async () => { + vi.mocked(selectDynamicLabelQueue).mockReturnValue(undefined); config = await createConfig(undefined, [ { ...baseRunner, matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, - enableDynamicLabels: false, + enableDynamicLabels: true, }, }, ]); @@ -296,7 +313,7 @@ describe('Dispatcher', () => { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:value'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); @@ -304,50 +321,20 @@ describe('Dispatcher', () => { expect(sendActionRequest).not.toHaveBeenCalled(); }); - it('keeps dynamic labels when the matched runner enables them and has no policy', async () => { + it('dispatches to the queue and labels returned by the provider selector', async () => { config = await createConfig(undefined, [ { ...baseRunner, + id: 'first', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, }, }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(201); - expect(sendActionRequest).toHaveBeenCalledWith( - expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'] }), - ); - }); - - it('skips a matching runner whose policy rejects the dynamic labels and uses the next compliant one', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'strict', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, { ...baseRunner, - id: 'permissive', + id: 'selected', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, @@ -355,61 +342,29 @@ describe('Dispatcher', () => { }, }, ]); + + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches) => ({ + queue: matches[1], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], + })); + const event = { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:requested'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ - queueId: 'permissive', - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + queueId: 'selected', + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], }), ); }); - it('rejects the job (202) when no runner accepts the policy', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'first', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, - { - ...baseRunner, - id: 'second', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: false, - }, - }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(202); - expect(sendActionRequest).not.toHaveBeenCalled(); - }); - it('forwards non-dynamic jobs as-is to the first match', async () => { config = await createConfig(undefined, [ { @@ -419,7 +374,6 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - awsDynamicLabelsPolicy: {}, }, }, ]); @@ -435,6 +389,7 @@ describe('Dispatcher', () => { expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ queueId: 'first', labels: ['self-hosted', 'linux'] }), ); + expect(selectDynamicLabelQueue).not.toHaveBeenCalled(); }); }); }); diff --git a/lambdas/functions/webhook/src/runners/dispatch.ts b/lambdas/functions/webhook/src/runners/dispatch.ts index 47c1f1bfc0..da6dc01221 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.ts @@ -1,11 +1,11 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { Response } from '../lambda'; import { RunnerMatcherConfig, sendActionRequest } from '../sqs'; import ValidationError from '../ValidationError'; import { ConfigDispatcher, ConfigWebhook, QueueSelectionStrategy } from '../ConfigLoader'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob, splitWorkflowJobLabels } from './labels'; const logger = createChildLogger('handler'); @@ -84,7 +84,7 @@ async function handleWorkflowJob( // Dynamic labels present: prefer the first provider-compliant queue. The // queue selection strategy applies to standard jobs only; dynamic-label jobs // always use the first compliant queue. - const dynamicTarget = selectAwsDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); + const dynamicTarget = selectDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); if (dynamicTarget) { targets = [dynamicTarget.queue]; diff --git a/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts new file mode 100644 index 0000000000..64b7507add --- /dev/null +++ b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts @@ -0,0 +1,61 @@ +import type { AwsDynamicLabelsPolicy } from '../contracts'; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${pattern}$`); +} + +function matchesAny(value: string, patterns: string[] | undefined): boolean { + if (!patterns || patterns.length === 0) return false; + return patterns.some((pattern) => globToRegExp(pattern).test(value)); +} + +function evaluateLabel(label: string, policy: AwsDynamicLabelsPolicy, labelPrefix: string): string | null { + const stripped = label.slice(labelPrefix.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? undefined : stripped.slice(colonIndex + 1); + + if (policy.blocked_keys?.includes(key)) { + return `key '${key}' is in blocked_keys`; + } + + const rule = policy.restricted_keys?.[key]; + if (!rule || value === undefined) return null; + + if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { + return `value '${value}' not in allowed list`; + } + if (rule.denied && matchesAny(value, rule.denied)) { + return `value '${value}' in denied list`; + } + if (rule.max !== undefined && rule.max !== null) { + const valueNumber = Number(value); + const maximum = Number(rule.max); + if (!Number.isFinite(valueNumber) || !Number.isFinite(maximum)) { + return `max set but value '${value}' or max '${rule.max}' is not numeric`; + } + if (valueNumber > maximum) { + return `value '${value}' exceeds max '${rule.max}'`; + } + } + + return null; +} + +export function violationsAgainstAwsDynamicLabelsPolicy( + labels: string[], + policy: AwsDynamicLabelsPolicy | null | undefined, + labelPrefix: string, +): { label: string; reason: string }[] { + if (!policy) return []; + + const violations: { label: string; reason: string }[] = []; + for (const label of labels) { + if (!label.startsWith(labelPrefix)) continue; + const reason = evaluateLabel(label, policy, labelPrefix); + if (reason) violations.push({ label, reason }); + } + return violations; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts index a9b919c7bd..8babbadd55 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts @@ -1,4 +1,5 @@ import type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; @@ -10,50 +11,6 @@ export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; */ export type Ec2DynamicLabelsPolicy = AwsDynamicLabelsPolicy; -function globToRegExp(glob: string): RegExp { - const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); - return new RegExp(`^${pattern}$`); -} - -function matchesAny(value: string, patterns: string[] | undefined): boolean { - if (!patterns || patterns.length === 0) return false; - return patterns.some((p) => globToRegExp(p).test(value)); -} - -function evaluateLabel(label: string, policy: Ec2DynamicLabelsPolicy): string | null { - const stripped = label.replace(/^ghr-ec2-/, ''); - const colonIdx = stripped.indexOf(':'); - const key = colonIdx === -1 ? stripped : stripped.slice(0, colonIdx); - const value = colonIdx === -1 ? undefined : stripped.slice(colonIdx + 1); - - if (policy.blocked_keys?.includes(key)) { - return `key '${key}' is in blocked_keys`; - } - - const rule = policy.restricted_keys?.[key]; - if (!rule) return null; - if (value === undefined) return null; - - if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { - return `value '${value}' not in allowed list`; - } - if (rule.denied && matchesAny(value, rule.denied)) { - return `value '${value}' in denied list`; - } - if (rule.max !== undefined && rule.max !== null) { - const valueNum = Number(value); - const maxNum = Number(rule.max); - if (!Number.isFinite(valueNum) || !Number.isFinite(maxNum)) { - return `max set but value '${value}' or max '${rule.max}' is not numeric`; - } - if (valueNum > maxNum) { - return `value '${value}' exceeds max '${rule.max}'`; - } - } - return null; -} - /** * Inspects the labels and returns the rejection reasons for any `ghr-ec2-*` * label that violates the policy. Non-`ghr-ec2-*` labels are ignored. @@ -62,12 +19,5 @@ export function violationsAgainstPolicy( labels: string[], policy: Ec2DynamicLabelsPolicy | null | undefined, ): { label: string; reason: string }[] { - if (!policy) return []; - const violations: { label: string; reason: string }[] = []; - for (const label of labels) { - if (!label.startsWith('ghr-ec2-')) continue; - const reason = evaluateLabel(label, policy); - if (reason) violations.push({ label, reason }); - } - return violations; + return violationsAgainstAwsDynamicLabelsPolicy(labels, policy, 'ghr-ec2-'); } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts index 400807554f..9b0cd07924 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts @@ -4,6 +4,64 @@ import type { RunnerMatcherConfig } from '../../../../contracts'; import { selectEc2DynamicLabelQueue } from './dynamic-labels'; describe('selectEc2DynamicLabelQueue', () => { + it('rejects dynamic labels when the queue disables them', () => { + const queue = runnerQueue('dynamic-labels-disabled'); + queue.matcherConfig.enableDynamicLabels = false; + + expect( + selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), + ).toBeUndefined(); + }); + + it('accepts dynamic labels when the queue has no policy', () => { + const queue = runnerQueue('no-policy'); + + expect(selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); + + it('skips a policy-rejected queue and returns the next compliant queue', () => { + const strictQueue = runnerQueue('strict'); + strictQueue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + const permissiveQueue = runnerQueue('permissive'); + + expect( + selectEc2DynamicLabelQueue( + [strictQueue, permissiveQueue], + ['self-hosted', 'linux'], + ['ghr-ec2-instance-type:t3.large'], + ), + ).toEqual({ + queue: permissiveQueue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); + + it('returns undefined when no queue accepts the dynamic labels', () => { + const strictQueue = runnerQueue('strict'); + strictQueue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + const disabledQueue = runnerQueue('disabled'); + disabledQueue.matcherConfig.enableDynamicLabels = false; + + expect( + selectEc2DynamicLabelQueue( + [strictQueue, disabledQueue], + ['self-hosted', 'linux'], + ['ghr-ec2-instance-type:t3.large'], + ), + ).toBeUndefined(); + }); + it('enforces a legacy EC2 dynamic labels policy when the new key is absent', () => { const queue = runnerQueue('legacy-ec2-policy'); queue.matcherConfig.ec2DynamicLabelsPolicy = { diff --git a/lambdas/libs/compute-providers/provider-types.test.ts b/lambdas/libs/compute-providers/provider-types.test.ts index 76111897ab..746274e5cc 100644 --- a/lambdas/libs/compute-providers/provider-types.test.ts +++ b/lambdas/libs/compute-providers/provider-types.test.ts @@ -23,9 +23,12 @@ describe('compute provider normalization', () => { expect(normalizeComputeProviderType(type)).toBe(expected); }); - it.each([[' Unknown '], ['microvm'], [null], [1]])('returns undefined for unsupported provider type %j', (type) => { - expect(normalizeComputeProviderType(type)).toBeUndefined(); - }); + it.each([[' Unknown '], ['unsupported-provider'], [null], [1]])( + 'returns undefined for unsupported provider type %j', + (type) => { + expect(normalizeComputeProviderType(type)).toBeUndefined(); + }, + ); }); describe('compute provider resolution', () => { @@ -38,7 +41,7 @@ describe('compute provider resolution', () => { expect(resolveComputeProviderType(type)).toBe(expected); }); - it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { + it.each([[' Unknown '], ['unsupported-provider'], [null], [1]])('rejects unsupported provider type %j', (type) => { expect(() => resolveComputeProviderType(type)).toThrow(`Unsupported compute provider type '${String(type)}'`); }); }); diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts b/lambdas/libs/compute-providers/webhook.test.ts similarity index 72% rename from lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts rename to lambdas/libs/compute-providers/webhook.test.ts index 790d4c2989..2007248b18 100644 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -1,14 +1,14 @@ -import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { describe, expect, it } from 'vitest'; -import type { RunnerMatcherConfig } from '../sqs'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; +import type { RunnerMatcherConfig } from './contracts'; +import type { ComputeProviderType } from './provider-types'; +import { selectDynamicLabelQueue } from './webhook'; -describe('selectAwsDynamicLabelQueue', () => { +describe('selectDynamicLabelQueue', () => { it('defaults queues without a provider to EC2 dynamic label handling', () => { const queue = runnerQueue('default-ec2'); - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ queue, labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], }); @@ -18,7 +18,7 @@ describe('selectAwsDynamicLabelQueue', () => { const queue = runnerQueue('normalized-ec2'); (queue as unknown as { computeProvider: string }).computeProvider = ' EC2 '; - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ queue, labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], }); @@ -30,7 +30,7 @@ describe('selectAwsDynamicLabelQueue', () => { const ec2Queue = runnerQueue('ec2'); expect( - selectAwsDynamicLabelQueue( + selectDynamicLabelQueue( [unsupportedQueue, ec2Queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'], @@ -46,7 +46,7 @@ describe('selectAwsDynamicLabelQueue', () => { (queue as unknown as { computeProvider: number }).computeProvider = 42; expect( - selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), + selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), ).toBeUndefined(); }); }); diff --git a/lambdas/libs/compute-providers/webhook.ts b/lambdas/libs/compute-providers/webhook.ts index ee80a54203..4c70c6a74c 100644 --- a/lambdas/libs/compute-providers/webhook.ts +++ b/lambdas/libs/compute-providers/webhook.ts @@ -1,8 +1,34 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + import { createComputeProviderRegistry } from './core'; -import type { WebhookProviderCapabilities } from './contracts'; +import type { DynamicLabelDispatchTarget, RunnerMatcherConfig, WebhookProviderCapabilities } from './contracts'; +import { normalizeComputeProviderType } from './provider-types'; import { enabledWebhookProviders } from './providers.config.webhook'; +const logger = createChildLogger('compute-provider-webhook'); + export const webhookProviderRegistry = createComputeProviderRegistry( enabledWebhookProviders.map((provider) => provider.createPlugin()), ); + +export function selectDynamicLabelQueue( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], +): DynamicLabelDispatchTarget | undefined { + for (const queue of matches) { + const provider = normalizeComputeProviderType(queue.computeProvider); + const dynamicLabels = provider ? webhookProviderRegistry.capability(provider, 'dynamicLabels') : undefined; + + if (!dynamicLabels) { + logger.warn(`Queue ${queue.id} has unsupported compute provider '${provider ?? String(queue.computeProvider)}'`); + continue; + } + + const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); + if (target) return target; + } + + return undefined; +} From a4ad8f508a575cbce7153ed00b04d1b624634610 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 21:16:34 +0200 Subject: [PATCH 02/21] refactor(compute-providers): resolve provider types strictly --- .../compute-providers/provider-types.test.ts | 43 ++++++------------- .../libs/compute-providers/provider-types.ts | 16 +++---- .../libs/compute-providers/webhook.test.ts | 27 +++--------- lambdas/libs/compute-providers/webhook.ts | 15 ++----- 4 files changed, 29 insertions(+), 72 deletions(-) diff --git a/lambdas/libs/compute-providers/provider-types.test.ts b/lambdas/libs/compute-providers/provider-types.test.ts index 746274e5cc..9f6f4a981e 100644 --- a/lambdas/libs/compute-providers/provider-types.test.ts +++ b/lambdas/libs/compute-providers/provider-types.test.ts @@ -1,11 +1,15 @@ import { describe, expect, it } from 'vitest'; -import { - defaultComputeProvider, - normalizeComputeProviderType, - resolveComputeProviderType, - computeProviderTypes, -} from './provider-types'; +import { computeProviderTypes, defaultComputeProvider, resolveComputeProviderType } from './provider-types'; + +const defaultProviderInputs = [undefined, '', ' '] as const; +const supportedProviderCases = computeProviderTypes.flatMap( + (provider) => + [ + [provider, provider], + [` ${provider.toUpperCase()} `, provider], + ] as const, +); describe('compute provider configuration', () => { it('defines an explicit default provider', () => { @@ -13,31 +17,12 @@ describe('compute provider configuration', () => { }); }); -describe('compute provider normalization', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('normalizes provider type %j to %j', (type, expected) => { - expect(normalizeComputeProviderType(type)).toBe(expected); +describe('compute provider resolution', () => { + it.each(defaultProviderInputs)('resolves default provider input %j', (type) => { + expect(resolveComputeProviderType(type)).toBe(defaultComputeProvider); }); - it.each([[' Unknown '], ['unsupported-provider'], [null], [1]])( - 'returns undefined for unsupported provider type %j', - (type) => { - expect(normalizeComputeProviderType(type)).toBeUndefined(); - }, - ); -}); - -describe('compute provider resolution', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('resolves provider type %j to %j', (type, expected) => { + it.each(supportedProviderCases)('resolves provider type %j to %j', (type, expected) => { expect(resolveComputeProviderType(type)).toBe(expected); }); diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts index dcac6c5769..64d7be8e5f 100644 --- a/lambdas/libs/compute-providers/provider-types.ts +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -4,21 +4,19 @@ export type ComputeProviderType = (typeof computeProviderTypes)[number]; export const defaultComputeProvider = 'ec2' satisfies ComputeProviderType; -export function normalizeComputeProviderType(type: unknown): ComputeProviderType | undefined { +export function resolveComputeProviderType(type: unknown): ComputeProviderType { if (type === undefined) return defaultComputeProvider; - if (typeof type !== 'string') return undefined; + if (typeof type !== 'string') { + throw new Error(`Unsupported compute provider type '${String(type)}'`); + } const normalizedType = type.trim().toLowerCase(); if (!normalizedType) return defaultComputeProvider; - return computeProviderTypes.find((computeProviderType) => computeProviderType === normalizedType); -} - -export function resolveComputeProviderType(type: unknown): ComputeProviderType { - const normalizedType = normalizeComputeProviderType(type); - if (!normalizedType) { + const computeProviderType = computeProviderTypes.find((provider) => provider === normalizedType); + if (!computeProviderType) { throw new Error(`Unsupported compute provider type '${String(type)}'`); } - return normalizedType; + return computeProviderType; } diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts index 2007248b18..b46e365246 100644 --- a/lambdas/libs/compute-providers/webhook.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -24,30 +24,13 @@ describe('selectDynamicLabelQueue', () => { }); }); - it('skips an unsupported provider strategy and selects the next supported queue', () => { - const unsupportedQueue = runnerQueue('unsupported-provider'); - (unsupportedQueue as unknown as { computeProvider: string }).computeProvider = 'unsupported'; - const ec2Queue = runnerQueue('ec2'); + it.each([['unsupported'], [42]])('throws for unsupported compute provider %j', (computeProvider) => { + const queue = runnerQueue('unsupported-provider'); + (queue as unknown as { computeProvider: unknown }).computeProvider = computeProvider; - expect( - selectDynamicLabelQueue( - [unsupportedQueue, ec2Queue], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toEqual({ - queue: ec2Queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('rejects a malformed non-string compute provider without throwing', () => { - const queue = runnerQueue('malformed-provider'); - (queue as unknown as { computeProvider: number }).computeProvider = 42; - - expect( + expect(() => selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + ).toThrow(`Unsupported compute provider type '${String(computeProvider)}'`); }); }); diff --git a/lambdas/libs/compute-providers/webhook.ts b/lambdas/libs/compute-providers/webhook.ts index 4c70c6a74c..2b3414777a 100644 --- a/lambdas/libs/compute-providers/webhook.ts +++ b/lambdas/libs/compute-providers/webhook.ts @@ -1,13 +1,9 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; - import { createComputeProviderRegistry } from './core'; import type { DynamicLabelDispatchTarget, RunnerMatcherConfig, WebhookProviderCapabilities } from './contracts'; -import { normalizeComputeProviderType } from './provider-types'; +import { resolveComputeProviderType } from './provider-types'; import { enabledWebhookProviders } from './providers.config.webhook'; -const logger = createChildLogger('compute-provider-webhook'); - export const webhookProviderRegistry = createComputeProviderRegistry( enabledWebhookProviders.map((provider) => provider.createPlugin()), ); @@ -18,13 +14,8 @@ export function selectDynamicLabelQueue( sanitizedGhrLabels: string[], ): DynamicLabelDispatchTarget | undefined { for (const queue of matches) { - const provider = normalizeComputeProviderType(queue.computeProvider); - const dynamicLabels = provider ? webhookProviderRegistry.capability(provider, 'dynamicLabels') : undefined; - - if (!dynamicLabels) { - logger.warn(`Queue ${queue.id} has unsupported compute provider '${provider ?? String(queue.computeProvider)}'`); - continue; - } + const provider = resolveComputeProviderType(queue.computeProvider); + const dynamicLabels = webhookProviderRegistry.capability(provider, 'dynamicLabels'); const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); if (target) return target; From b6084a36311da75c7e548aaf94e650949b2587bb Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 13 Aug 2026 00:28:54 +0200 Subject: [PATCH 03/21] refactor(compute-providers): centralize dynamic label selection --- .../ec2/src/webhook/dynamic-labels.test.ts | 76 ++++--------- .../aws/ec2/src/webhook/dynamic-labels.ts | 36 +----- lambdas/libs/compute-providers/contracts.ts | 11 +- .../compute-providers/dynamic-labels.test.ts | 12 ++ .../libs/compute-providers/dynamic-labels.ts | 8 ++ .../libs/compute-providers/registry.test.ts | 2 +- .../templates/provider/provider.test.ts | 2 +- .../templates/provider/webhook.ts | 6 +- .../libs/compute-providers/webhook.test.ts | 106 ++++++++++++++---- lambdas/libs/compute-providers/webhook.ts | 71 +++++++++--- 10 files changed, 195 insertions(+), 135 deletions(-) create mode 100644 lambdas/libs/compute-providers/dynamic-labels.test.ts create mode 100644 lambdas/libs/compute-providers/dynamic-labels.ts diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts index 9b0cd07924..99c1844a5f 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts @@ -1,65 +1,29 @@ import { describe, expect, it } from 'vitest'; import type { RunnerMatcherConfig } from '../../../../contracts'; -import { selectEc2DynamicLabelQueue } from './dynamic-labels'; +import { ec2DynamicLabelProvider } from './dynamic-labels'; -describe('selectEc2DynamicLabelQueue', () => { - it('rejects dynamic labels when the queue disables them', () => { - const queue = runnerQueue('dynamic-labels-disabled'); - queue.matcherConfig.enableDynamicLabels = false; - - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); - }); - - it('accepts dynamic labels when the queue has no policy', () => { +describe('ec2DynamicLabelProvider', () => { + it('returns no violations when the queue has no policy', () => { const queue = runnerQueue('no-policy'); - expect(selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); + expect(getViolations(queue)).toEqual([]); }); - it('skips a policy-rejected queue and returns the next compliant queue', () => { + it('returns violations for labels rejected by the policy', () => { const strictQueue = runnerQueue('strict'); strictQueue.matcherConfig.awsDynamicLabelsPolicy = { restricted_keys: { 'instance-type': { allowed: ['m5.*'] }, }, }; - const permissiveQueue = runnerQueue('permissive'); - expect( - selectEc2DynamicLabelQueue( - [strictQueue, permissiveQueue], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toEqual({ - queue: permissiveQueue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('returns undefined when no queue accepts the dynamic labels', () => { - const strictQueue = runnerQueue('strict'); - strictQueue.matcherConfig.awsDynamicLabelsPolicy = { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, + expect(getViolations(strictQueue)).toEqual([ + { + label: 'ghr-ec2-instance-type:t3.large', + reason: "value 't3.large' not in allowed list", }, - }; - const disabledQueue = runnerQueue('disabled'); - disabledQueue.matcherConfig.enableDynamicLabels = false; - - expect( - selectEc2DynamicLabelQueue( - [strictQueue, disabledQueue], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toBeUndefined(); + ]); }); it('enforces a legacy EC2 dynamic labels policy when the new key is absent', () => { @@ -68,9 +32,7 @@ describe('selectEc2DynamicLabelQueue', () => { blocked_keys: ['instance-type'], }; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('falls back to the legacy EC2 dynamic labels policy when the new policy is null', () => { @@ -80,9 +42,7 @@ describe('selectEc2DynamicLabelQueue', () => { }; queue.matcherConfig.awsDynamicLabelsPolicy = null; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('prefers a configured AWS dynamic labels policy over the legacy policy', () => { @@ -94,13 +54,17 @@ describe('selectEc2DynamicLabelQueue', () => { blocked_keys: [], }; - expect(selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); + expect(getViolations(queue)).toEqual([]); }); }); +function getViolations(queue: RunnerMatcherConfig) { + return ec2DynamicLabelProvider.getViolations({ + queue, + labels: ['ghr-ec2-instance-type:t3.large'], + }); +} + function runnerQueue(id: string): RunnerMatcherConfig { return { id, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts index 6ddf5b8fbb..5e671da189 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts @@ -1,12 +1,10 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import type { DynamicLabelDispatchTarget, DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; +import type { DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; import { violationsAgainstPolicy } from './dynamic-labels-policy'; const logger = createChildLogger('handler'); -export type Ec2DynamicLabelDispatchTarget = DynamicLabelDispatchTarget; - function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { const hasLegacyEc2DynamicLabelsPolicy = Object.prototype.hasOwnProperty.call( queue.matcherConfig, @@ -23,36 +21,6 @@ function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { return queue.matcherConfig.awsDynamicLabelsPolicy; } -export function selectEc2DynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): Ec2DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - if (!queue.matcherConfig.enableDynamicLabels) { - logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); - continue; - } - - const violations = violationsAgainstPolicy(sanitizedGhrLabels, resolveEc2DynamicLabelsPolicy(queue)); - if (violations.length === 0) { - return { - queue, - labels: [...nonGhrLabels, ...sanitizedGhrLabels], - }; - } - - for (const violation of violations) { - logger.warn( - `Queue ${queue.id}: dynamic label '${violation.label}' does not match policy (${violation.reason}); trying next match`, - ); - } - } - - return undefined; -} - export const ec2DynamicLabelProvider: DynamicLabelProvider = { - selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => - selectEc2DynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), + getViolations: ({ queue, labels }) => violationsAgainstPolicy(labels, resolveEc2DynamicLabelsPolicy(queue)), }; diff --git a/lambdas/libs/compute-providers/contracts.ts b/lambdas/libs/compute-providers/contracts.ts index 617789ec10..85e99f3949 100644 --- a/lambdas/libs/compute-providers/contracts.ts +++ b/lambdas/libs/compute-providers/contracts.ts @@ -43,12 +43,13 @@ export interface DynamicLabelDispatchTarget { labels: string[]; } +export interface DynamicLabelViolation { + label: string; + reason: string; +} + export interface DynamicLabelProvider { - selectQueue(input: { - queue: RunnerMatcherConfig; - nonGhrLabels: string[]; - sanitizedGhrLabels: string[]; - }): DynamicLabelDispatchTarget | undefined; + getViolations(input: { queue: RunnerMatcherConfig; labels: string[] }): DynamicLabelViolation[]; } export interface ControlPlaneProviderCapabilities { diff --git a/lambdas/libs/compute-providers/dynamic-labels.test.ts b/lambdas/libs/compute-providers/dynamic-labels.test.ts new file mode 100644 index 0000000000..0eacc9cf62 --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from 'vitest'; + +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; +import { computeProviderTypes } from './provider-types'; + +it.each(computeProviderTypes)('returns labels belonging to providers other than %s', (provider) => { + const providerLabels = computeProviderTypes.map((type) => `ghr-${type}-size:large`); + + expect(dynamicLabelsForOtherProvider(providerLabels, provider)).toEqual( + providerLabels.filter((label) => !label.startsWith(`ghr-${provider}-`)), + ); +}); diff --git a/lambdas/libs/compute-providers/dynamic-labels.ts b/lambdas/libs/compute-providers/dynamic-labels.ts new file mode 100644 index 0000000000..97db9517d3 --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.ts @@ -0,0 +1,8 @@ +import { computeProviderTypes } from './provider-types'; +import type { ComputeProviderType } from './provider-types'; + +export function dynamicLabelsForOtherProvider(labels: string[], provider: ComputeProviderType): string[] { + return labels.filter((label) => + computeProviderTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), + ); +} diff --git a/lambdas/libs/compute-providers/registry.test.ts b/lambdas/libs/compute-providers/registry.test.ts index 3c95dcaca4..93227831cd 100644 --- a/lambdas/libs/compute-providers/registry.test.ts +++ b/lambdas/libs/compute-providers/registry.test.ts @@ -33,6 +33,6 @@ it('exposes every configured provider through both capability registries', () => unmarkOrphan: expect.any(Function), terminate: expect.any(Function), }); - expect(webhookProviderRegistry.capability(type, 'dynamicLabels').selectQueue).toEqual(expect.any(Function)); + expect(webhookProviderRegistry.capability(type, 'dynamicLabels').getViolations).toEqual(expect.any(Function)); } }); diff --git a/lambdas/libs/compute-providers/templates/provider/provider.test.ts b/lambdas/libs/compute-providers/templates/provider/provider.test.ts index 816b2f9cfc..2644fc4f2a 100644 --- a/lambdas/libs/compute-providers/templates/provider/provider.test.ts +++ b/lambdas/libs/compute-providers/templates/provider/provider.test.ts @@ -29,5 +29,5 @@ it('exposes every compute provider capability from its compute-provider entry po terminate: expect.any(Function), }); expect(webhookPlugin.type).toBe(webhookProvider.type); - expect(webhookPlugin.capabilities.dynamicLabels.selectQueue).toEqual(expect.any(Function)); + expect(webhookPlugin.capabilities.dynamicLabels.getViolations).toEqual(expect.any(Function)); }); diff --git a/lambdas/libs/compute-providers/templates/provider/webhook.ts b/lambdas/libs/compute-providers/templates/provider/webhook.ts index 86e59da7b3..31c522c588 100644 --- a/lambdas/libs/compute-providers/templates/provider/webhook.ts +++ b/lambdas/libs/compute-providers/templates/provider/webhook.ts @@ -3,10 +3,10 @@ import type { ComputeProviderPlugin } from '../../core'; import type { DynamicLabelProvider, WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; export const templateDynamicLabelProvider: DynamicLabelProvider = { - selectQueue: (input) => { + getViolations: (input) => { void input; - // Return a dispatch target when this provider accepts the requested dynamic labels. - return undefined; + // Return violations for dynamic labels this provider does not accept. + return []; }, }; diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts index b46e365246..4316c3aca5 100644 --- a/lambdas/libs/compute-providers/webhook.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -1,44 +1,106 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; -import type { RunnerMatcherConfig } from './contracts'; +import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; import type { ComputeProviderType } from './provider-types'; -import { selectDynamicLabelQueue } from './webhook'; +import { createDynamicLabelQueueSelector } from './webhook'; -describe('selectDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { - const queue = runnerQueue('default-ec2'); +describe('createDynamicLabelQueueSelector', () => { + it('returns the first queue accepted by its provider', () => { + const queue = runnerQueue('accepted'); + const { selectQueue } = selector(); - expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + expect(selectQueue([queue], ['self-hosted', 'linux'], ['ghr-test-size:large'])).toEqual({ queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-test-size:large'], }); }); - it('normalizes compute provider casing and surrounding whitespace', () => { - const queue = runnerQueue('normalized-ec2'); - (queue as unknown as { computeProvider: string }).computeProvider = ' EC2 '; + it('skips queues that disable dynamic labels', () => { + const disabledQueue = runnerQueue('disabled'); + disabledQueue.matcherConfig.enableDynamicLabels = false; + const enabledQueue = runnerQueue('enabled'); + const { getViolations, selectQueue } = selector(); - expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + expect(selectQueue([disabledQueue, enabledQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: enabledQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: enabledQueue, labels: ['ghr-test-size:large'] }); + }); + + it('skips queues whose provider reports violations', () => { + const rejectedQueue = runnerQueue('rejected'); + const acceptedQueue = runnerQueue('accepted'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, + }); + + expect(selectQueue([rejectedQueue, acceptedQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: acceptedQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + }); + + it('returns undefined when every provider reports violations', () => { + const queue = runnerQueue('rejected'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, }); + + expect(selectQueue([queue], ['self-hosted'], ['ghr-test-size:large'])).toBeUndefined(); }); - it.each([['unsupported'], [42]])('throws for unsupported compute provider %j', (computeProvider) => { - const queue = runnerQueue('unsupported-provider'); - (queue as unknown as { computeProvider: unknown }).computeProvider = computeProvider; + /* TODO: Re-enable this scenario when the MicroVM provider is added. + it('skips EC2 and selects the MicroVM queue for MicroVM override labels', () => { + const ec2Queue = runnerQueue('ec2'); + const microvmQueue = runnerQueue('microvm'); + const imageVersionLabel = 'ghr-microvm-image-version:3.0'; + const { getViolations, selectQueue } = selector({ + providerByQueue: { ec2: 'ec2', microvm: 'microvm' }, + labelsForOtherProvider: (labels, provider) => + provider === 'ec2' ? labels.filter((label) => label.startsWith('ghr-microvm-')) : [], + }); - expect(() => - selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toThrow(`Unsupported compute provider type '${String(computeProvider)}'`); + expect(selectQueue([ec2Queue, microvmQueue], ['self-hosted', 'linux'], [imageVersionLabel])).toEqual({ + queue: microvmQueue, + labels: ['self-hosted', 'linux', imageVersionLabel], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: microvmQueue, labels: [imageVersionLabel] }); }); + */ }); -function runnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { +function selector(options?: { + providerByQueue?: Record; + violationsByQueue?: Record; + labelsForOtherProvider?: (labels: string[], provider: ComputeProviderType) => string[]; +}) { + const getViolations = vi.fn(({ queue }) => { + return options?.violationsByQueue?.[queue.id] ?? []; + }); + + return { + getViolations, + selectQueue: createDynamicLabelQueueSelector({ + resolveProvider: (queue) => ({ + type: options?.providerByQueue?.[queue.id] ?? 'ec2', + dynamicLabels: { getViolations }, + }), + dynamicLabelsForOtherProvider: options?.labelsForOtherProvider ?? (() => []), + }), + }; +} + +function runnerQueue(id: string): RunnerMatcherConfig { return { id, arn: `arn:${id}`, - computeProvider, matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, diff --git a/lambdas/libs/compute-providers/webhook.ts b/lambdas/libs/compute-providers/webhook.ts index 2b3414777a..1aa0a7b5e6 100644 --- a/lambdas/libs/compute-providers/webhook.ts +++ b/lambdas/libs/compute-providers/webhook.ts @@ -1,25 +1,70 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + import { createComputeProviderRegistry } from './core'; -import type { DynamicLabelDispatchTarget, RunnerMatcherConfig, WebhookProviderCapabilities } from './contracts'; +import type { + DynamicLabelDispatchTarget, + DynamicLabelProvider, + RunnerMatcherConfig, + WebhookProviderCapabilities, +} from './contracts'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; import { resolveComputeProviderType } from './provider-types'; import { enabledWebhookProviders } from './providers.config.webhook'; +const logger = createChildLogger('handler'); + export const webhookProviderRegistry = createComputeProviderRegistry( enabledWebhookProviders.map((provider) => provider.createPlugin()), ); -export function selectDynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - const provider = resolveComputeProviderType(queue.computeProvider); - const dynamicLabels = webhookProviderRegistry.capability(provider, 'dynamicLabels'); +export function createDynamicLabelQueueSelector(dependencies: { + resolveProvider(queue: RunnerMatcherConfig): { type: TProvider; dynamicLabels: DynamicLabelProvider }; + dynamicLabelsForOtherProvider(labels: string[], provider: TProvider): string[]; +}) { + return ( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], + ): DynamicLabelDispatchTarget | undefined => { + for (const queue of matches) { + const { type: provider, dynamicLabels } = dependencies.resolveProvider(queue); + + if (!queue.matcherConfig.enableDynamicLabels) { + logger.warn( + `Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`, + ); + continue; + } - const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); - if (target) return target; - } + const labelsForOtherProvider = dependencies.dynamicLabelsForOtherProvider(sanitizedGhrLabels, provider); + if (labelsForOtherProvider.length > 0) { + logger.warn(`Queue ${queue.id}: dynamic labels target another compute provider; trying next match`, { + dynamicLabels: labelsForOtherProvider, + }); + continue; + } - return undefined; + const violations = dynamicLabels.getViolations({ queue, labels: sanitizedGhrLabels }); + if (violations.length === 0) { + return { queue, labels: [...nonGhrLabels, ...sanitizedGhrLabels] }; + } + + for (const violation of violations) { + logger.warn( + `Queue ${queue.id}: dynamic label '${violation.label}' is not accepted (${violation.reason}); trying next match`, + ); + } + } + + return undefined; + }; } + +export const selectDynamicLabelQueue = createDynamicLabelQueueSelector({ + resolveProvider: (queue) => { + const type = resolveComputeProviderType(queue.computeProvider); + return { type, dynamicLabels: webhookProviderRegistry.capability(type, 'dynamicLabels') }; + }, + dynamicLabelsForOtherProvider, +}); From 4db38886c5d27f988ff448f895b8b6a491210085 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 15:27:34 +0200 Subject: [PATCH 04/21] test(compute-providers): cover dynamic label selection --- .../compute-providers/dynamic-labels.test.ts | 10 +++-- .../libs/compute-providers/dynamic-labels.ts | 12 ++++-- .../libs/compute-providers/webhook.test.ts | 39 ++++++++++++++++++- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/lambdas/libs/compute-providers/dynamic-labels.test.ts b/lambdas/libs/compute-providers/dynamic-labels.test.ts index 0eacc9cf62..e93b9fa264 100644 --- a/lambdas/libs/compute-providers/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/dynamic-labels.test.ts @@ -1,10 +1,12 @@ import { expect, it } from 'vitest'; -import { dynamicLabelsForOtherProvider } from './dynamic-labels'; -import { computeProviderTypes } from './provider-types'; +import { createDynamicLabelsForOtherProvider } from './dynamic-labels'; -it.each(computeProviderTypes)('returns labels belonging to providers other than %s', (provider) => { - const providerLabels = computeProviderTypes.map((type) => `ghr-${type}-size:large`); +const providerTypes = ['alpha', 'beta'] as const; +const dynamicLabelsForOtherProvider = createDynamicLabelsForOtherProvider(providerTypes); + +it.each(providerTypes)('returns labels belonging to providers other than %s', (provider) => { + const providerLabels = providerTypes.map((type) => `ghr-${type}-size:large`); expect(dynamicLabelsForOtherProvider(providerLabels, provider)).toEqual( providerLabels.filter((label) => !label.startsWith(`ghr-${provider}-`)), diff --git a/lambdas/libs/compute-providers/dynamic-labels.ts b/lambdas/libs/compute-providers/dynamic-labels.ts index 97db9517d3..8ac72757c8 100644 --- a/lambdas/libs/compute-providers/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/dynamic-labels.ts @@ -1,8 +1,12 @@ import { computeProviderTypes } from './provider-types'; import type { ComputeProviderType } from './provider-types'; -export function dynamicLabelsForOtherProvider(labels: string[], provider: ComputeProviderType): string[] { - return labels.filter((label) => - computeProviderTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), - ); +export function createDynamicLabelsForOtherProvider(providerTypes: readonly TProvider[]) { + return (labels: string[], provider: TProvider): string[] => + labels.filter((label) => + providerTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), + ); } + +export const dynamicLabelsForOtherProvider = + createDynamicLabelsForOtherProvider(computeProviderTypes); diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts index 4316c3aca5..f3120a3b59 100644 --- a/lambdas/libs/compute-providers/webhook.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -2,7 +2,44 @@ import { describe, expect, it, vi } from 'vitest'; import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; import type { ComputeProviderType } from './provider-types'; -import { createDynamicLabelQueueSelector } from './webhook'; +import { createDynamicLabelQueueSelector, selectDynamicLabelQueue } from './webhook'; + +describe('selectDynamicLabelQueue', () => { + it('defaults queues without a provider to EC2 dynamic label handling', () => { + const queue = runnerQueue('default-ec2'); + + expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); + + it('normalizes compute provider casing and surrounding whitespace', () => { + const queue = runnerQueue('normalized-ec2'); + (queue as unknown as { computeProvider: string }).computeProvider = ' EC2 '; + + expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); + + it.each([ + ['unsupported string', 'unsupported-provider'], + ['non-string', 42], + ])('strictly rejects an %s compute provider', (_description, computeProvider) => { + const invalidQueue = runnerQueue('invalid-provider'); + (invalidQueue as unknown as { computeProvider: unknown }).computeProvider = computeProvider; + + expect(() => + selectDynamicLabelQueue( + [invalidQueue, runnerQueue('valid-ec2')], + ['self-hosted', 'linux'], + ['ghr-ec2-instance-type:t3.large'], + ), + ).toThrow(`Unsupported compute provider type '${String(computeProvider)}'`); + }); +}); describe('createDynamicLabelQueueSelector', () => { it('returns the first queue accepted by its provider', () => { From 383d921e93ca14ee69ce3a0b9ab7a05f17b9c8e0 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 15:51:50 +0200 Subject: [PATCH 05/21] test(compute-providers): share webhook provider contract --- .../compute-providers/aws/ec2/webhook.test.ts | 7 ++ .../test/webhook-provider-contract.ts | 58 ++++++++++++++++ lambdas/libs/compute-providers/tsconfig.json | 2 +- .../libs/compute-providers/webhook.test.ts | 66 ++++++------------- 4 files changed, 87 insertions(+), 46 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/ec2/webhook.test.ts create mode 100644 lambdas/libs/compute-providers/test/webhook-provider-contract.ts diff --git a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts new file mode 100644 index 0000000000..d6557d3c88 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts @@ -0,0 +1,7 @@ +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; +import { provider } from './webhook'; + +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-ec2-instance-type:t3.large'], +}); diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts new file mode 100644 index 0000000000..a970e5f10e --- /dev/null +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig, WebhookProviderModule } from '../contracts'; +import { defaultComputeProvider } from '../provider-types'; +import type { ComputeProviderType } from '../provider-types'; +import { selectDynamicLabelQueue } from '../webhook'; + +interface WebhookProviderContractOptions { + provider: WebhookProviderModule; + acceptedDynamicLabels: readonly [string, ...string[]]; +} + +export function defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels, +}: WebhookProviderContractOptions): void { + const nonGhrLabels = ['self-hosted', 'linux']; + const dynamicLabels = [...acceptedDynamicLabels]; + + function expectProviderSelected(queue: RunnerMatcherConfig) { + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toEqual({ + queue, + labels: [...nonGhrLabels, ...dynamicLabels], + }); + } + + describe(`${provider.type} webhook provider contract`, () => { + it('selects an explicitly configured provider through the production registry', () => { + expectProviderSelected(runnerQueue(`${provider.type}-configured`, provider.type)); + }); + + it('normalizes provider configuration before registry selection', () => { + const queue = runnerQueue(`${provider.type}-normalized`); + (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; + + expectProviderSelected(queue); + }); + + if (provider.type === defaultComputeProvider) { + it('selects the default provider when the queue omits provider configuration', () => { + expectProviderSelected(runnerQueue(`${provider.type}-default`)); + }); + } + }); +} + +function runnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + computeProvider, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/tsconfig.json b/lambdas/libs/compute-providers/tsconfig.json index 52d55867fe..51beb73b87 100644 --- a/lambdas/libs/compute-providers/tsconfig.json +++ b/lambdas/libs/compute-providers/tsconfig.json @@ -1,5 +1,5 @@ { "extends": "../../tsconfig.json", - "include": ["*.ts", "core/**/*", "aws/**/*", "templates/**/*"], + "include": ["*.ts", "core/**/*", "aws/**/*", "templates/**/*", "test/**/*"], "exclude": ["aws/**/*.test.ts"] } diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts index f3120a3b59..983da7d028 100644 --- a/lambdas/libs/compute-providers/webhook.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -1,29 +1,14 @@ import { describe, expect, it, vi } from 'vitest'; import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; -import type { ComputeProviderType } from './provider-types'; +import { createDynamicLabelsForOtherProvider } from './dynamic-labels'; import { createDynamicLabelQueueSelector, selectDynamicLabelQueue } from './webhook'; -describe('selectDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { - const queue = runnerQueue('default-ec2'); - - expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('normalizes compute provider casing and surrounding whitespace', () => { - const queue = runnerQueue('normalized-ec2'); - (queue as unknown as { computeProvider: string }).computeProvider = ' EC2 '; - - expect(selectDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); +const testProviderTypes = ['alpha', 'beta'] as const; +type TestProviderType = (typeof testProviderTypes)[number]; +const dynamicLabelsForOtherProvider = createDynamicLabelsForOtherProvider(testProviderTypes); +describe('selectDynamicLabelQueue', () => { it.each([ ['unsupported string', 'unsupported-provider'], ['non-string', 42], @@ -31,13 +16,9 @@ describe('selectDynamicLabelQueue', () => { const invalidQueue = runnerQueue('invalid-provider'); (invalidQueue as unknown as { computeProvider: unknown }).computeProvider = computeProvider; - expect(() => - selectDynamicLabelQueue( - [invalidQueue, runnerQueue('valid-ec2')], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toThrow(`Unsupported compute provider type '${String(computeProvider)}'`); + expect(() => selectDynamicLabelQueue([invalidQueue], [], [])).toThrow( + `Unsupported compute provider type '${String(computeProvider)}'`, + ); }); }); @@ -92,31 +73,26 @@ describe('createDynamicLabelQueueSelector', () => { expect(selectQueue([queue], ['self-hosted'], ['ghr-test-size:large'])).toBeUndefined(); }); - /* TODO: Re-enable this scenario when the MicroVM provider is added. - it('skips EC2 and selects the MicroVM queue for MicroVM override labels', () => { - const ec2Queue = runnerQueue('ec2'); - const microvmQueue = runnerQueue('microvm'); - const imageVersionLabel = 'ghr-microvm-image-version:3.0'; + it('selects the queue targeted by provider-specific labels', () => { + const alphaQueue = runnerQueue('alpha'); + const betaQueue = runnerQueue('beta'); + const betaLabel = 'ghr-beta-size:large'; const { getViolations, selectQueue } = selector({ - providerByQueue: { ec2: 'ec2', microvm: 'microvm' }, - labelsForOtherProvider: (labels, provider) => - provider === 'ec2' ? labels.filter((label) => label.startsWith('ghr-microvm-')) : [], + providerByQueue: { alpha: 'alpha', beta: 'beta' }, }); - expect(selectQueue([ec2Queue, microvmQueue], ['self-hosted', 'linux'], [imageVersionLabel])).toEqual({ - queue: microvmQueue, - labels: ['self-hosted', 'linux', imageVersionLabel], + expect(selectQueue([alphaQueue, betaQueue], ['self-hosted', 'linux'], [betaLabel])).toEqual({ + queue: betaQueue, + labels: ['self-hosted', 'linux', betaLabel], }); expect(getViolations).toHaveBeenCalledOnce(); - expect(getViolations).toHaveBeenCalledWith({ queue: microvmQueue, labels: [imageVersionLabel] }); + expect(getViolations).toHaveBeenCalledWith({ queue: betaQueue, labels: [betaLabel] }); }); - */ }); function selector(options?: { - providerByQueue?: Record; + providerByQueue?: Record; violationsByQueue?: Record; - labelsForOtherProvider?: (labels: string[], provider: ComputeProviderType) => string[]; }) { const getViolations = vi.fn(({ queue }) => { return options?.violationsByQueue?.[queue.id] ?? []; @@ -124,12 +100,12 @@ function selector(options?: { return { getViolations, - selectQueue: createDynamicLabelQueueSelector({ + selectQueue: createDynamicLabelQueueSelector({ resolveProvider: (queue) => ({ - type: options?.providerByQueue?.[queue.id] ?? 'ec2', + type: options?.providerByQueue?.[queue.id] ?? 'alpha', dynamicLabels: { getViolations }, }), - dynamicLabelsForOtherProvider: options?.labelsForOtherProvider ?? (() => []), + dynamicLabelsForOtherProvider, }), }; } From 98e159716c3f39d2b03ae1cbc8240e2d391191c0 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 16:18:05 +0200 Subject: [PATCH 06/21] refactor(compute-providers): simplify provider label filtering --- .../compute-providers/dynamic-labels.test.ts | 5 ++--- .../libs/compute-providers/dynamic-labels.ts | 17 ++++++++--------- lambdas/libs/compute-providers/webhook.test.ts | 6 +++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lambdas/libs/compute-providers/dynamic-labels.test.ts b/lambdas/libs/compute-providers/dynamic-labels.test.ts index e93b9fa264..88aa39689c 100644 --- a/lambdas/libs/compute-providers/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/dynamic-labels.test.ts @@ -1,14 +1,13 @@ import { expect, it } from 'vitest'; -import { createDynamicLabelsForOtherProvider } from './dynamic-labels'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; const providerTypes = ['alpha', 'beta'] as const; -const dynamicLabelsForOtherProvider = createDynamicLabelsForOtherProvider(providerTypes); it.each(providerTypes)('returns labels belonging to providers other than %s', (provider) => { const providerLabels = providerTypes.map((type) => `ghr-${type}-size:large`); - expect(dynamicLabelsForOtherProvider(providerLabels, provider)).toEqual( + expect(dynamicLabelsForOtherProvider(providerLabels, provider, providerTypes)).toEqual( providerLabels.filter((label) => !label.startsWith(`ghr-${provider}-`)), ); }); diff --git a/lambdas/libs/compute-providers/dynamic-labels.ts b/lambdas/libs/compute-providers/dynamic-labels.ts index 8ac72757c8..3c72d77966 100644 --- a/lambdas/libs/compute-providers/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/dynamic-labels.ts @@ -1,12 +1,11 @@ import { computeProviderTypes } from './provider-types'; -import type { ComputeProviderType } from './provider-types'; -export function createDynamicLabelsForOtherProvider(providerTypes: readonly TProvider[]) { - return (labels: string[], provider: TProvider): string[] => - labels.filter((label) => - providerTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), - ); +export function dynamicLabelsForOtherProvider( + labels: string[], + provider: string, + providerTypes: readonly string[] = computeProviderTypes, +): string[] { + return labels.filter((label) => + providerTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), + ); } - -export const dynamicLabelsForOtherProvider = - createDynamicLabelsForOtherProvider(computeProviderTypes); diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts index 983da7d028..7ec7343f97 100644 --- a/lambdas/libs/compute-providers/webhook.test.ts +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -1,12 +1,11 @@ import { describe, expect, it, vi } from 'vitest'; import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; -import { createDynamicLabelsForOtherProvider } from './dynamic-labels'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; import { createDynamicLabelQueueSelector, selectDynamicLabelQueue } from './webhook'; const testProviderTypes = ['alpha', 'beta'] as const; type TestProviderType = (typeof testProviderTypes)[number]; -const dynamicLabelsForOtherProvider = createDynamicLabelsForOtherProvider(testProviderTypes); describe('selectDynamicLabelQueue', () => { it.each([ @@ -105,7 +104,8 @@ function selector(options?: { type: options?.providerByQueue?.[queue.id] ?? 'alpha', dynamicLabels: { getViolations }, }), - dynamicLabelsForOtherProvider, + dynamicLabelsForOtherProvider: (labels, provider) => + dynamicLabelsForOtherProvider(labels, provider, testProviderTypes), }), }; } From a301a7d026aea19da1802d8cba8fbb435d0900a8 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 16:24:15 +0200 Subject: [PATCH 07/21] test(compute-providers): cover disabled dynamic labels --- .../compute-providers/test/webhook-provider-contract.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index a970e5f10e..759329880b 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -29,6 +29,13 @@ export function defineWebhookProviderContractTests { + const queue = runnerQueue(`${provider.type}-disabled`, provider.type); + queue.matcherConfig.enableDynamicLabels = false; + + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); + }); + it('normalizes provider configuration before registry selection', () => { const queue = runnerQueue(`${provider.type}-normalized`); (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; From 3602f4812df21f29956f112e129b46015cc2a839 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 16:32:05 +0200 Subject: [PATCH 08/21] test(compute-providers): cover AWS dynamic label policy --- lambdas/libs/compute-providers/aws/ec2/webhook.test.ts | 5 +++++ .../compute-providers/test/webhook-provider-contract.ts | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts index d6557d3c88..7fa5d4ffa5 100644 --- a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts @@ -4,4 +4,9 @@ import { provider } from './webhook'; defineWebhookProviderContractTests({ provider, acceptedDynamicLabels: ['ghr-ec2-instance-type:t3.large'], + applyRejectingPolicy: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['instance-type'], + }; + }, }); diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index 759329880b..50651cbfe0 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -8,11 +8,13 @@ import { selectDynamicLabelQueue } from '../webhook'; interface WebhookProviderContractOptions { provider: WebhookProviderModule; acceptedDynamicLabels: readonly [string, ...string[]]; + applyRejectingPolicy(queue: RunnerMatcherConfig): void; } export function defineWebhookProviderContractTests({ provider, acceptedDynamicLabels, + applyRejectingPolicy, }: WebhookProviderContractOptions): void { const nonGhrLabels = ['self-hosted', 'linux']; const dynamicLabels = [...acceptedDynamicLabels]; @@ -36,6 +38,13 @@ export function defineWebhookProviderContractTests { + const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); + applyRejectingPolicy(queue); + + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); + }); + it('normalizes provider configuration before registry selection', () => { const queue = runnerQueue(`${provider.type}-normalized`); (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; From ac35170850e3879ce09dcc6eaec1525e3c0541e6 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 14 Aug 2026 16:38:41 +0200 Subject: [PATCH 09/21] test(compute-providers): cover restricted AWS policy --- .../compute-providers/aws/ec2/webhook.test.ts | 25 +++++++++++++++---- .../test/webhook-provider-contract.ts | 21 ++++++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts index 7fa5d4ffa5..755831fb91 100644 --- a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts @@ -4,9 +4,24 @@ import { provider } from './webhook'; defineWebhookProviderContractTests({ provider, acceptedDynamicLabels: ['ghr-ec2-instance-type:t3.large'], - applyRejectingPolicy: (queue) => { - queue.matcherConfig.awsDynamicLabelsPolicy = { - blocked_keys: ['instance-type'], - }; - }, + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['instance-type'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + }, + }, + ], }); diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index 50651cbfe0..dd4e3097b0 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -5,16 +5,21 @@ import { defaultComputeProvider } from '../provider-types'; import type { ComputeProviderType } from '../provider-types'; import { selectDynamicLabelQueue } from '../webhook'; +interface RejectingPolicyCase { + name: string; + apply(queue: RunnerMatcherConfig): void; +} + interface WebhookProviderContractOptions { provider: WebhookProviderModule; acceptedDynamicLabels: readonly [string, ...string[]]; - applyRejectingPolicy(queue: RunnerMatcherConfig): void; + rejectingPolicies: readonly [RejectingPolicyCase, ...RejectingPolicyCase[]]; } export function defineWebhookProviderContractTests({ provider, acceptedDynamicLabels, - applyRejectingPolicy, + rejectingPolicies, }: WebhookProviderContractOptions): void { const nonGhrLabels = ['self-hosted', 'linux']; const dynamicLabels = [...acceptedDynamicLabels]; @@ -38,12 +43,14 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); - applyRejectingPolicy(queue); + for (const policy of rejectingPolicies) { + it(`skips the provider when its ${policy.name} policy rejects the labels`, () => { + const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); + policy.apply(queue); - expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); - }); + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); + }); + } it('normalizes provider configuration before registry selection', () => { const queue = runnerQueue(`${provider.type}-normalized`); From b38c721992ea86f195177e1e177a44e79b70153d Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 20:41:02 +0200 Subject: [PATCH 10/21] feat(compute-providers): add MicroVM API foundations --- .../microvm/src/control-plane/config.test.ts | 71 ++++ .../aws/microvm/src/control-plane/config.ts | 88 +++++ .../src/control-plane/microvms.test.ts | 311 ++++++++++++++++++ .../aws/microvm/src/control-plane/microvms.ts | 223 +++++++++++++ .../aws/microvm/src/environment.d.ts | 15 + lambdas/libs/compute-providers/package.json | 1 + .../libs/compute-providers/provider-types.ts | 2 +- lambdas/yarn.lock | 289 ++++++++++++++++ 8 files changed, 999 insertions(+), 1 deletion(-) create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts new file mode 100644 index 0000000000..ce692a1ab4 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; + +const cleanEnv = process.env; + +beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + delete process.env.MICROVM_IMAGE_VERSION; + delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; + delete process.env.MICROVM_LOG_GROUP; +}); + +describe('loadMicrovmProviderConfig', () => { + it('loads required values and applies optional defaults', () => { + expect(loadMicrovmProviderConfig()).toEqual({ + imageIdentifier: process.env.MICROVM_IMAGE_ARN, + imageVersion: undefined, + executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, + ingressNetworkConnectors: undefined, + egressNetworkConnectors: undefined, + maximumDurationInSeconds: 3600, + logging: undefined, + }); + }); + + it('loads versions, logging, duration, and either connector list format', () => { + process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; + process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + + expect(loadMicrovmProviderConfig()).toMatchObject({ + imageVersion: '3.0', + ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], + egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + maximumDurationInSeconds: 1200, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, + }); + }); + + it.each([ + ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ])('requires %s', (environmentVariable, expectedName) => { + delete process.env[environmentVariable]; + + expect(() => loadMicrovmProviderConfig()).toThrow( + `${expectedName} must be configured for the MicroVM compute provider`, + ); + }); + + it.each(['0', '28801', '1.5', 'invalid'])('rejects invalid maximum duration %s', (duration) => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = duration; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and 28800', + ); + }); + + it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts new file mode 100644 index 0000000000..7c0a7662b9 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -0,0 +1,88 @@ +import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +const DEFAULT_MAXIMUM_DURATION_IN_SECONDS = 3600; +const MAXIMUM_DURATION_IN_SECONDS = 28800; + +export interface MicrovmProviderConfig { + egressNetworkConnectors?: string[]; + executionRoleArn: string; + imageIdentifier: string; + imageVersion?: string; + ingressNetworkConnectors?: string[]; + logging?: Logging; + maximumDurationInSeconds: number; +} + +function requiredEnvironmentValue(name: string, value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`${name} must be configured for the MicroVM compute provider`); + } + return trimmed; +} + +function optionalEnvironmentValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return undefined; + + let connectors: unknown; + try { + connectors = configuredValue.startsWith('[') + ? JSON.parse(configuredValue) + : configuredValue.split(',').map((connector) => connector.trim()); + } catch (error) { + throw new Error(`${name} must be a JSON array or comma-separated list`, { cause: error }); + } + + if ( + !Array.isArray(connectors) || + connectors.length === 0 || + connectors.some((connector) => typeof connector !== 'string' || connector.trim().length === 0) + ) { + throw new Error(`${name} must contain one or more non-empty connector ARNs`); + } + + return connectors.map((connector) => connector.trim()); +} + +function parseMaximumDuration(value: string | undefined): number { + if (!optionalEnvironmentValue(value)) return DEFAULT_MAXIMUM_DURATION_IN_SECONDS; + + const maximumDurationInSeconds = Number(value); + if ( + !Number.isInteger(maximumDurationInSeconds) || + maximumDurationInSeconds < 1 || + maximumDurationInSeconds > MAXIMUM_DURATION_IN_SECONDS + ) { + throw new Error( + `MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, + ); + } + + return maximumDurationInSeconds; +} + +export function loadMicrovmProviderConfig(): MicrovmProviderConfig { + const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); + + return { + imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), + imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), + ingressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_INGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS, + ), + egressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_EGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, + ), + maximumDurationInSeconds: parseMaximumDuration(process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS), + logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts new file mode 100644 index 0000000000..7d7199a3cc --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -0,0 +1,311 @@ +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + ListTagsCommand, + RunMicrovmCommand, + TagResourceCommand, + TerminateMicrovmCommand, + UntagResourceCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MicrovmProviderConfig } from './config'; +import { + isRetryableMicrovmError, + listMicrovmRunners, + microvmArn, + microvmBootTimeExceeded, + runMicrovmRunner, + tagMicrovm, + terminateMicrovm, + untagMicrovm, +} from './microvms'; + +const mockMicrovmClient = mockClient(LambdaMicrovmsClient); +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const config: MicrovmProviderConfig = { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 1200, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, +}; + +beforeEach(() => { + mockMicrovmClient.reset(); + vi.useRealTimers(); + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; +}); + +describe('microvmArn', () => { + it('derives the MicroVM resource ARN from its image ARN', () => { + expect(microvmArn(imageArn, 'mvm-123')).toBe('arn:aws:lambda:eu-west-1:123456789012:microvm:mvm-123'); + expect(microvmArn(imageArn.replace('arn:aws:', 'arn:aws-us-gov:'), 'mvm-456')).toContain('arn:aws-us-gov:lambda:'); + }); + + it('rejects image names that cannot identify a customer MicroVM resource', () => { + expect(() => microvmArn('runner', 'mvm-123')).toThrow( + 'MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN', + ); + }); +}); + +describe('runMicrovmRunner', () => { + it('launches and tags a managed runner', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123' }); + mockMicrovmClient.on(TagResourceCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{"version":1}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).resolves.toBe('mvm-123'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: config.executionRoleArn, + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 1200, + logging: config.logging, + runHookPayload: '{"version":1}', + clientToken: expect.any(String), + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:created_by': 'scale-up-lambda', + 'ghr:environment': 'unit-test', + 'ghr:Owner': 'Codertocat', + 'ghr:Type': 'Org', + }, + }); + }); + + it('rejects a launch response without an ID', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'pool-lambda', + }), + ).rejects.toThrow('RunMicrovm returned no microvmId'); + }); + + it('terminates a new runner when required tags cannot be applied', async () => { + const tagError = new Error('tag failed'); + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); + mockMicrovmClient.on(TagResourceCommand).rejects(tagError); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).rejects.toThrow('tag failed'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-untagged', + }); + }); + + it('preserves the tag error when cleanup also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); + mockMicrovmClient.on(TagResourceCommand).rejects(new Error('tag failed')); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).rejects.toThrow('tag failed'); + }); +}); + +describe('listMicrovmRunners', () => { + it('paginates active MicroVMs and filters them by management tags', async () => { + const startedAt = new Date('2026-08-06T10:00:00.000Z'); + mockMicrovmClient + .on(ListMicrovmsCommand) + .resolvesOnce({ + nextToken: 'page-2', + items: [ + { microvmId: 'mvm-managed', imageArn, imageVersion: '3.0', startedAt, state: 'RUNNING' }, + { microvmId: 'mvm-terminated', imageArn, imageVersion: '3.0', startedAt, state: 'TERMINATED' }, + ], + }) + .resolvesOnce({ + items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], + }); + mockMicrovmClient + .on(ListTagsCommand) + .resolvesOnce({ + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:environment': 'unit-test', + 'ghr:Owner': 'Codertocat', + 'ghr:Type': 'Org', + 'ghr:github_runner_id': '42', + 'ghr:bypass-removal': 'true', + }, + }) + .resolvesOnce({ Tags: { 'ghr:Application': 'another-application' } }); + + await expect( + listMicrovmRunners({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }), + ).resolves.toEqual([ + { + id: 'mvm-managed', + imageArn, + launchTime: startedAt, + owner: 'Codertocat', + type: 'Org', + orphan: false, + githubRunnerId: '42', + bypassRemoval: true, + state: 'RUNNING', + }, + ]); + + expect(mockMicrovmClient).toHaveReceivedNthCommandWith(2, ListMicrovmsCommand, { + maxResults: 50, + nextToken: 'page-2', + }); + }); + + it('applies environment, owner, type, and orphan filters after loading tags', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { + microvmId: 'mvm-filtered', + imageArn, + imageVersion: '3.0', + startedAt: new Date(), + state: 'SUSPENDED', + }, + ], + }); + mockMicrovmClient.on(ListTagsCommand).resolves({ + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:environment': 'other', + 'ghr:Owner': 'Other', + 'ghr:Type': 'Repo', + }, + }); + + await expect(listMicrovmRunners({ environment: 'unit-test' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true })).resolves.toEqual([]); + }); + + it('skips a MicroVM that terminates before its tags can be read', async () => { + const resourceNotFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-gone', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + }); + mockMicrovmClient.on(ListTagsCommand).rejects(resourceNotFound); + + await expect(listMicrovmRunners()).resolves.toEqual([]); + }); + + it('surfaces unexpected tag lookup failures', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + }); + mockMicrovmClient.on(ListTagsCommand).rejects(new Error('list tags failed')); + + await expect(listMicrovmRunners()).rejects.toThrow('list tags failed'); + }); +}); + +describe('MicroVM lifecycle helpers', () => { + it('tags, untags, and terminates a MicroVM', async () => { + mockMicrovmClient.on(TagResourceCommand).resolves({}); + mockMicrovmClient.on(UntagResourceCommand).resolves({}); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await tagMicrovm(imageArn, 'mvm-123', { key: 'value' }); + await untagMicrovm(imageArn, 'mvm-123', ['key']); + await terminateMicrovm('mvm-123'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + Tags: { key: 'value' }, + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(UntagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + TagKeys: ['key'], + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); + }); + + it('evaluates the configured boot window', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-06T10:10:00.000Z')); + + expect(microvmBootTimeExceeded({})).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:06:00.000Z') })).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:04:00.000Z') })).toBe(true); + }); +}); + +describe('isRetryableMicrovmError', () => { + it.each(['ConflictException', 'InternalServerException', 'ServiceQuotaExceededException', 'ThrottlingException'])( + 'classifies %s as retryable', + (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }, + ); + + it('classifies server, throttling, network, and nested failures as retryable', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); + expect(isRetryableMicrovmError(Object.assign(new Error('throttle'), { $metadata: { httpStatusCode: 429 } }))).toBe( + true, + ); + expect(isRetryableMicrovmError(Object.assign(new Error('network'), { code: 'ECONNRESET' }))).toBe(true); + expect( + isRetryableMicrovmError( + Object.assign(new Error('outer'), { cause: Object.assign(new Error(), { code: 'ETIMEDOUT' }) }), + ), + ).toBe(true); + }); + + it('does not retry configuration, unknown, or non-error failures', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('invalid'), { name: 'ValidationException' }))).toBe(false); + expect(isRetryableMicrovmError(new Error('unknown'))).toBe(false); + expect(isRetryableMicrovmError('failure')).toBe(false); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts new file mode 100644 index 0000000000..edc4a3775b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -0,0 +1,223 @@ +import { randomUUID } from 'node:crypto'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + ListTagsCommand, + RunMicrovmCommand, + TagResourceCommand, + TerminateMicrovmCommand, + UntagResourceCommand, +} from '@aws-sdk/client-lambda-microvms'; +import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +import type { LambdaRunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; +import type { MicrovmProviderConfig } from './config'; + +const logger = createChildLogger('microvm-runners'); + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_TAG_VALUE = 'github-action-runner'; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerInfo extends RunnerInfo { + imageArn?: string; + state?: MicrovmState; +} + +export interface RunMicrovmRunnerInput { + config: MicrovmProviderConfig; + environment: string; + runHookPayload: string; + runnerOwner: string; + runnerType: RunnerType; + source: LambdaRunnerSource; +} + +interface AwsErrorLike extends Error { + cause?: unknown; + code?: string; + $fault?: 'client' | 'server'; + $metadata?: { httpStatusCode?: number }; +} + +const RETRYABLE_ERROR_NAMES = new Set([ + 'ConflictException', + 'InternalServerException', + 'RequestTimeout', + 'RequestTimeoutException', + 'ResourceConflictException', + 'ServiceException', + 'ServiceQuotaExceededException', + 'Throttling', + 'ThrottlingException', + 'TooManyRequestsException', +]); + +const RETRYABLE_NETWORK_ERROR_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'ETIMEDOUT', +]); + +function microvmClient(): LambdaMicrovmsClient { + return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); +} + +export function microvmArn(imageArn: string, microvmId: string): string { + const match = /^arn:([^:]+):lambda:([^:]+):([0-9]{12}):microvm-image:.+$/.exec(imageArn); + if (!match) { + throw new Error(`MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN: ${imageArn}`); + } + + const [, partition, region, accountId] = match; + return `arn:${partition}:lambda:${region}:${accountId}:microvm:${microvmId}`; +} + +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + const commandInput: RunMicrovmCommandInput = { + imageIdentifier: input.config.imageIdentifier, + imageVersion: input.config.imageVersion, + executionRoleArn: input.config.executionRoleArn, + ingressNetworkConnectors: input.config.ingressNetworkConnectors, + egressNetworkConnectors: input.config.egressNetworkConnectors, + maximumDurationInSeconds: input.config.maximumDurationInSeconds, + logging: input.config.logging, + runHookPayload: input.runHookPayload, + clientToken: randomUUID(), + }; + + logger.debug('Launching Lambda MicroVM runner', { + imageIdentifier: commandInput.imageIdentifier, + imageVersion: commandInput.imageVersion, + maximumDurationInSeconds: commandInput.maximumDurationInSeconds, + }); + + const response = await microvmClient().send(new RunMicrovmCommand(commandInput)); + if (!response.microvmId) { + throw new Error('RunMicrovm returned no microvmId'); + } + + try { + await tagMicrovm(input.config.imageIdentifier, response.microvmId, { + [APPLICATION_TAG]: APPLICATION_TAG_VALUE, + 'ghr:created_by': input.source, + 'ghr:environment': input.environment, + 'ghr:Owner': input.runnerOwner, + 'ghr:Type': input.runnerType, + }); + } catch (error) { + logger.error(`Failed to tag new MicroVM runner '${response.microvmId}', terminating it`, { error }); + await terminateMicrovm(response.microvmId).catch((terminationError) => { + logger.error(`Failed to terminate untagged MicroVM runner '${response.microvmId}'`, { + error: terminationError, + }); + }); + throw error; + } + + return response.microvmId; +} + +export async function listMicrovmRunners(filters: ListRunnerFilters = {}): Promise { + const client = microvmClient(); + const items: MicrovmItem[] = []; + let nextToken: string | undefined; + + do { + const response = await client.send( + new ListMicrovmsCommand({ + maxResults: 50, + nextToken, + }), + ); + items.push(...(response.items ?? [])); + nextToken = response.nextToken; + } while (nextToken); + + const runners: MicrovmRunnerInfo[] = []; + for (const item of items) { + if (!item.microvmId || !item.imageArn || !item.state || !ACTIVE_STATES.has(item.state)) continue; + + let tags: Record; + try { + tags = + (await client.send(new ListTagsCommand({ Resource: microvmArn(item.imageArn, item.microvmId) }))).Tags ?? {}; + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') continue; + throw error; + } + + if (tags[APPLICATION_TAG] !== APPLICATION_TAG_VALUE) continue; + if (filters.environment !== undefined && tags['ghr:environment'] !== filters.environment) continue; + if (filters.runnerType !== undefined && tags['ghr:Type'] !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && tags['ghr:Owner'] !== filters.runnerOwner) continue; + if (filters.orphan && tags['ghr:orphan'] !== 'true') continue; + + runners.push({ + id: item.microvmId, + imageArn: item.imageArn, + launchTime: item.startedAt, + owner: tags['ghr:Owner'], + type: tags['ghr:Type'] as RunnerInfo['type'], + orphan: tags['ghr:orphan'] === 'true', + githubRunnerId: tags['ghr:github_runner_id'], + bypassRemoval: tags['ghr:bypass-removal'] === 'true', + state: item.state, + }); + } + + return runners; +} + +export async function tagMicrovm(imageArn: string, microvmId: string, tags: Record): Promise { + await microvmClient().send( + new TagResourceCommand({ + Resource: microvmArn(imageArn, microvmId), + Tags: tags, + }), + ); +} + +export async function untagMicrovm(imageArn: string, microvmId: string, tagKeys: string[]): Promise { + await microvmClient().send( + new UntagResourceCommand({ + Resource: microvmArn(imageArn, microvmId), + TagKeys: tagKeys, + }), + ); +} + +export async function terminateMicrovm(microvmId: string): Promise { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); +} + +export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { + if (!runner.launchTime) return false; + + const bootTimeInMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5'); + return runner.launchTime.getTime() + bootTimeInMinutes * 60_000 < Date.now(); +} + +export function isRetryableMicrovmError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + const awsError = error as AwsErrorLike; + if (RETRYABLE_ERROR_NAMES.has(awsError.name)) return true; + + const statusCode = awsError.$metadata?.httpStatusCode; + if ( + awsError.$fault === 'server' || + statusCode === 429 || + (statusCode !== undefined && statusCode >= 500) || + (awsError.code !== undefined && RETRYABLE_NETWORK_ERROR_CODES.has(awsError.code)) + ) { + return true; + } + + return awsError.cause !== undefined && awsError.cause !== error ? isRetryableMicrovmError(awsError.cause) : false; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts new file mode 100644 index 0000000000..91c1931f83 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -0,0 +1,15 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + MICROVM_EGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_EXECUTION_ROLE_ARN: string; + MICROVM_IMAGE_ARN: string; + MICROVM_IMAGE_VERSION: string | undefined; + MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_LOG_GROUP: string | undefined; + MICROVM_MAXIMUM_DURATION_IN_SECONDS: string | undefined; + } + } +} diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index 9d39fd294a..a6ebc12163 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -27,6 +27,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-lambda-microvms": "^3.1074.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts index 64d7be8e5f..087f61de71 100644 --- a/lambdas/libs/compute-providers/provider-types.ts +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -1,4 +1,4 @@ -export const computeProviderTypes = ['ec2'] as const; +export const computeProviderTypes = ['ec2', 'microvm'] as const; export type ComputeProviderType = (typeof computeProviderTypes)[number]; diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 56ae435c2c..4f60fc7f3a 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -148,6 +148,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-lambda-microvms": "npm:^3.1074.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -439,6 +440,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-lambda-microvms@npm:^3.1074.0": + version: 3.1104.0 + resolution: "@aws-sdk/client-lambda-microvms@npm:3.1104.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-node": "npm:^3.972.78" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/219ad52f822def4caa4a20d8d91d46a1b78e6726363a145be86c37cdeef4e4c13653e8a59ada67154146c6c2554e2c12944efad35c689850bf6f72f2d55246f4 + languageName: node + linkType: hard + "@aws-sdk/client-s3@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-s3@npm:3.1014.0" @@ -620,6 +637,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.6": + version: 3.977.6 + resolution: "@aws-sdk/core@npm:3.977.6" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4d743603bb41aeed426e2928be0947202191c341f9fbefe9ea347b0b4b7154b1ea94189d01c8abf3b03b9635449e2e7c268bd67379ea2294b9f49a61b909b9af + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -643,6 +676,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/547bcac01ac0912d0e42bb11f7d51bafcf2eaab1db35a098bea2be322211a86457ea60455a5294e58081c32376c240b07e66f81946a9be30e9722f723c6eaac2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -661,6 +707,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.69": + version: 3.972.69 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.69" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6e4cf9628919163a2a9784bf8618bc85a8c0ba7056813bedb9758c04eb3b36663f5099cfad329f89ac86c4e408bb3d0698ee7cff7e4a61c8a0335ab98078d678 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -683,6 +744,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-login": "npm:^3.972.74" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/84646fee1c61e31b2052d902559ecf163c1d00558ecdc21d77b396250527348b9ebba324d0bf8ffee4b3e45476c691de502e6faad3d39d1f7420eee5d326c7c5 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -699,6 +781,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1ab9996accb61bccbdaefae023e9befab9e5062435370a37f672089457dc13c485d9d2fee6926381672bdb32143c00266b1aa5913b18ef4873584907835b3a92 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -719,6 +815,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.78": + version: 3.972.78 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.78" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-ini": "npm:^3.973.12" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2b6e5bd455a3c2b530a884a0c5919bb7d2d91941b655a56351b957de038d318c1d42b86674e20893ee7dab6db6ea32c4653bce9b1d3ca98ac803d6b49a948343 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -733,6 +848,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0381c39f171df2119791b03647545ab5084f6a8d2c227c5d3c5bfa9db027d0566102b6322bc09075c0779b0f0aa88ae1ca7bbdc773d8415d8dd8f163e69e45ea + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -749,6 +877,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.11": + version: 3.973.11 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.11" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/token-providers": "npm:3.1103.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d6df0ae72009c2f74f1c7f12e41c0a7b395ba1860d4f9f1554fd8fbc3b5f0c1be64c83aacd2b329561e27844520ae0b57bb268265f5e7850b1d0fb769455d7a1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -764,6 +907,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.73": + version: 3.972.73 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.73" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a7bee06b4200ff04141d4ce07d49d69f24b55b57f3aab929b445a9d9ea70f043d0b09fca8b95872d2b92df6837b771aeb14b03ca784d17ce1ee871b14173558c + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -1002,6 +1159,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.41": + version: 3.997.41 + resolution: "@aws-sdk/nested-clients@npm:3.997.41" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/fe1a84bb58675a24ecd0ce3b7bcaf1a456494f10c1a9dd5b55bd268be6713f83bd3c3d3dadee6bb51a53bc24ee18b2b0882d6741bcbabc224feeae97454fdb4a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1029,6 +1202,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1044,6 +1229,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1103.0": + version: 3.1103.0 + resolution: "@aws-sdk/token-providers@npm:3.1103.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5f86aa221e537b8a3fd11ed76ac025935f8859cc62b3af293abd759b8ca3aa390c17f6716723c08056b3373997a17bbdee2ff568eab5049bf0a372d35893b48d + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1054,6 +1253,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1139,6 +1348,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1146,6 +1365,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.23.5, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4404,6 +4630,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4417,6 +4653,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4485,6 +4732,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4650,6 +4908,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4735,6 +5004,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4768,6 +5048,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" From 7bd96562dcb0fdd4d113771331b7b944f45a3f47 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 20:41:42 +0200 Subject: [PATCH 11/21] feat(compute-providers): add MicroVM control-plane provider --- .../aws/microvm/control-plane.ts | 25 +++ .../microvm/src/control-plane/pool.test.ts | 112 ++++++++++ .../aws/microvm/src/control-plane/pool.ts | 65 ++++++ .../src/control-plane/runner-config.test.ts | 191 ++++++++++++++++++ .../src/control-plane/runner-config.ts | 109 ++++++++++ .../src/control-plane/scale-down.test.ts | 74 +++++++ .../microvm/src/control-plane/scale-down.ts | 28 +++ .../src/control-plane/scale-up.test.ts | 114 +++++++++++ .../aws/microvm/src/control-plane/scale-up.ts | 77 +++++++ .../aws/microvm/src/dynamic-labels.test.ts | 73 +++++++ .../aws/microvm/src/dynamic-labels.ts | 90 +++++++++ lambdas/libs/compute-providers/package.json | 3 +- .../providers.config.control-plane.ts | 3 +- 13 files changed, 962 insertions(+), 2 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/microvm/control-plane.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts diff --git a/lambdas/libs/compute-providers/aws/microvm/control-plane.ts b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts new file mode 100644 index 0000000000..d6287ca1e1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts @@ -0,0 +1,25 @@ +import type { ComputeProviderPlugin, CreateStartRunnerConfig } from '../../core'; + +import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; +import type {} from './src/environment'; +import { createMicrovmPoolProvider } from './src/control-plane/pool'; +import { createMicrovmScaleDownProvider } from './src/control-plane/scale-down'; +import { createMicrovmScaleUpProvider } from './src/control-plane/scale-up'; + +export function createMicrovmControlPlanePlugin( + createStartRunnerConfig: CreateStartRunnerConfig, +): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { + pool: () => createMicrovmPoolProvider(createStartRunnerConfig), + scaleUp: () => createMicrovmScaleUpProvider(createStartRunnerConfig), + scaleDown: createMicrovmScaleDownProvider, + }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmControlPlanePlugin, +} satisfies ControlPlaneProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts new file mode 100644 index 0000000000..8f46818b50 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts @@ -0,0 +1,112 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import type { MicrovmRunnerInfo } from './microvms'; +import { calculateMicrovmPoolSize, createMicrovmPoolProvider } from './pool'; +import { createMicrovmRunners } from './runner-config'; + +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), +})); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +function runner(id: string, state: MicrovmRunnerInfo['state']): MicrovmRunnerInfo { + return { id, state, owner: 'Codertocat', type: 'Org' }; +} + +function githubRunnerConfig(): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('calculateMicrovmPoolSize', () => { + it('counts online idle running runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-idle', 'RUNNING')], + new Map([['mvm-idle', { busy: false, status: 'online' }]]), + ), + ).toBe(1); + }); + + it('optionally counts online busy runners', () => { + const runners = [runner('mvm-busy', 'RUNNING')]; + const statuses = new Map([['mvm-busy', { busy: true, status: 'online' }]]); + + expect(calculateMicrovmPoolSize(runners, statuses)).toBe(0); + expect(calculateMicrovmPoolSize(runners, statuses, true)).toBe(1); + }); + + it('counts pending runners only during their boot window', () => { + const runners = [runner('mvm-pending', 'PENDING')]; + vi.mocked(microvmBootTimeExceeded).mockReturnValueOnce(false).mockReturnValueOnce(true); + + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(1); + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(0); + }); + + it('does not count suspended or offline runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-suspended', 'SUSPENDED'), runner('mvm-offline', 'RUNNING')], + new Map([['mvm-offline', { busy: false, status: 'offline' }]]), + ), + ).toBe(0); + }); +}); + +describe('createMicrovmPoolProvider', () => { + it('lists managed MicroVMs and returns successfully created IDs', async () => { + const provider = createMicrovmPoolProvider(createStartRunnerConfig); + const input = { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + }; + + await expect(provider.listRunners(input)).resolves.toEqual([]); + expect(listMicrovmRunners).toHaveBeenCalledWith(input); + + await expect( + provider.createRunners({ + githubRunnerConfig: githubRunnerConfig(), + numberOfRunners: 1, + githubInstallationClient: githubClient, + }), + ).resolves.toEqual(['mvm-1']); + expect(createMicrovmRunners).toHaveBeenCalledWith( + expect.any(Object), + 1, + githubClient, + createStartRunnerConfig, + 'pool-lambda', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts new file mode 100644 index 0000000000..8deed5562d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts @@ -0,0 +1,65 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { + CreatePoolRunnersInput, + CreateStartRunnerConfig, + ListPoolRunnersInput, + PoolComputeProvider, + RunnerStatus, +} from '../../../../core'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +const logger = createChildLogger('microvm-pool'); + +async function listMicrovmPoolRunners(input: ListPoolRunnersInput): Promise { + return await listMicrovmRunners(input); +} + +async function createMicrovmPoolRunners( + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + const result = await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'pool-lambda', + ); + return result.instances; +} + +export function calculateMicrovmPoolSize( + runners: MicrovmRunnerInfo[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + let availableRunners = 0; + + for (const runner of runners) { + const status = runnerStatus.get(runner.id); + if (runner.state === 'RUNNING' && status?.status === 'online' && (!status.busy || includeBusyRunners)) { + availableRunners++; + logger.debug(`MicroVM runner ${runner.id} is online and counted as part of the pool`); + } else if (runner.state === 'PENDING' && !microvmBootTimeExceeded(runner)) { + availableRunners++; + logger.info(`MicroVM runner ${runner.id} is still booting and counted as part of the pool`); + } else { + logger.debug(`MicroVM runner ${runner.id} is not available and is not counted as part of the pool`); + } + } + + return availableRunners; +} + +export function createMicrovmPoolProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + listRunners: listMicrovmPoolRunners, + countAvailableRunners: calculateMicrovmPoolSize, + createRunners: (input) => createMicrovmPoolRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts new file mode 100644 index 0000000000..84afb5be3b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -0,0 +1,191 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; +import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + isRetryableMicrovmError: vi.fn(), + runMicrovmRunner: vi.fn(), + tagMicrovm: vi.fn(), + terminateMicrovm: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const githubClient = {} as Octokit; +const createStartRunnerConfig = vi.fn(); + +function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/github-action-runners/unit-test/token', + ssmConfigPath: '/github-action-runners/unit-test/config', + ssmParameterStoreTags: [], + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 1200, + }); + vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); + vi.mocked(tagMicrovm).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); + vi.mocked(isRetryableMicrovmError).mockReturnValue(false); + createStartRunnerConfig.mockResolvedValue([]); +}); + +describe('createMicrovmRunHookPayload', () => { + it('contains only the versioned SSM prefix contract', () => { + expect(JSON.parse(createMicrovmRunHookPayload('/runner/token'))).toEqual({ + version: 1, + runnerConfigSsmPath: '/runner/token', + }); + }); +}); + +describe('createMicrovmRunners', () => { + it.each([{ ephemeral: false }, { enableJitConfig: false }])( + 'rejects unsupported runner configuration %j', + async (overrides) => { + await expect( + createMicrovmRunners(runnerConfig(overrides), 2, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 2 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }, + ); + + it('requires an SSM token path', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmTokenPath: '' }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + }); + + it('classifies invalid provider configuration as non-retryable', async () => { + vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { + throw new Error('missing image'); + }); + + await expect( + createMicrovmRunners(runnerConfig(), 3, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 3 }); + }); + + it('launches each MicroVM and delivers its JIT configuration', async () => { + vi.mocked(runMicrovmRunner).mockResolvedValueOnce('mvm-1').mockResolvedValueOnce('mvm-2'); + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: `github-${runnerIds[0]}`, runnerLabels: [] }); + return []; + }); + + await expect( + createMicrovmRunners(runnerConfig(), 2, githubClient, createStartRunnerConfig, 'pool-lambda'), + ).resolves.toEqual({ instances: ['mvm-1', 'mvm-2'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { + config: expect.objectContaining({ imageIdentifier: imageArn }), + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'pool-lambda', + }); + expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); + const options = createStartRunnerConfig.mock.calls[0][3]; + expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); + expect(tagMicrovm).toHaveBeenNthCalledWith(1, imageArn, 'mvm-1', { + 'ghr:github_runner_id': 'github-mvm-1', + }); + }); + + it('applies dynamic labels to the RunMicrovm configuration and metadata tags', async () => { + const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; + const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: 'github-mvm-1', runnerLabels: [] }); + return []; + }); + + await createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda', { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }); + + expect(runMicrovmRunner).toHaveBeenCalledWith({ + config: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 7200, + }, + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }); + expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { + 'ghr:github_runner_id': 'github-mvm-1', + }); + }); + + it('retries a JIT setup failure even when runner cleanup fails', async () => { + createStartRunnerConfig.mockResolvedValue(['mvm-1']); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); + + it.each([ + [true, { instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }], + [false, { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }], + ])('classifies launch failures with retryable=%s', async (retryable, expected) => { + vi.mocked(runMicrovmRunner).mockRejectedValue(new Error('launch failed')); + vi.mocked(isRetryableMicrovmError).mockReturnValue(retryable); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual(expected); + }); + + it('attempts cleanup when setup throws after launch', async () => { + createStartRunnerConfig.mockRejectedValue(new Error('JIT setup failed')); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts new file mode 100644 index 0000000000..ca3497dd0e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -0,0 +1,109 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Octokit } from '@octokit/rest'; + +import type { + CreateGitHubRunnerConfig, + CreateRunnerResult, + CreateStartRunnerConfig, + LambdaRunnerSource, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; + +const logger = createChildLogger('microvm-runner-config'); + +export interface MicrovmRunHookPayloadV1 { + runnerConfigSsmPath: string; + version: 1; +} + +export function createMicrovmRunHookPayload(ssmTokenPath: string): string { + return JSON.stringify({ + version: 1, + runnerConfigSsmPath: ssmTokenPath, + } satisfies MicrovmRunHookPayloadV1); +} + +export async function createMicrovmRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + numberOfRunners: number, + githubInstallationClient: Octokit, + createStartRunnerConfig: CreateStartRunnerConfig, + source: LambdaRunnerSource, + overrides: MicrovmDynamicLabelOverrides = {}, +): Promise { + if (!githubRunnerConfig.ephemeral || !githubRunnerConfig.enableJitConfig) { + logger.error('Lambda MicroVM runners require ephemeral runners with JIT configuration enabled'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + if (!githubRunnerConfig.ssmTokenPath?.trim()) { + logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + let config; + try { + config = { ...loadMicrovmProviderConfig(), ...overrides }; + } catch (error) { + logger.error('Invalid Lambda MicroVM provider configuration', { error }); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + const result: CreateRunnerResult = { + instances: [], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }; + const runHookPayload = createMicrovmRunHookPayload(githubRunnerConfig.ssmTokenPath); + + for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { + let microvmId: string | undefined; + try { + microvmId = await runMicrovmRunner({ + config, + environment: process.env.ENVIRONMENT, + runHookPayload, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + source, + }); + + const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { + getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await tagMicrovm(config.imageIdentifier, runnerId, { + 'ghr:github_runner_id': metadata.githubRunnerId, + }); + }, + }); + + if (failedRunnerIds.includes(microvmId)) { + await terminateMicrovm(microvmId).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { + error: terminationError, + }); + }); + result.retryableErrorCount++; + } else { + result.instances.push(microvmId); + } + } catch (error) { + if (microvmId) { + await terminateMicrovm(microvmId).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { + error: terminationError, + }); + }); + } + + const retryable = isRetryableMicrovmError(error); + logger.error('Failed to create Lambda MicroVM runner', { error, retryable }); + if (retryable) result.retryableErrorCount++; + else result.nonRetryableErrorCount++; + } + } + + return result; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts new file mode 100644 index 0000000000..613364e7d2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; +import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; +import { createMicrovmScaleDownProvider } from './scale-down'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), + tagMicrovm: vi.fn(), + terminateMicrovm: vi.fn(), + untagMicrovm: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const providerConfig = { + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 1200, +}; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(tagMicrovm).mockResolvedValue(); + vi.mocked(untagMicrovm).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); +}); + +describe('createMicrovmScaleDownProvider', () => { + it('lists active and orphan runners through provider filters', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.list('unit-test', true); + + expect(listMicrovmRunners).toHaveBeenNthCalledWith(1, { + environment: 'unit-test', + orphan: undefined, + }); + expect(listMicrovmRunners).toHaveBeenNthCalledWith(2, { + environment: 'unit-test', + orphan: true, + }); + }); + + it('uses the listed image ARN when marking, unmarking, and terminating runners', async () => { + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-1', imageArn: overrideImageArn, owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.markOrphan('mvm-1'); + await provider.unmarkOrphan('mvm-1'); + await provider.terminate('mvm-1'); + + expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { 'ghr:orphan': 'true' }); + expect(untagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', ['ghr:orphan']); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); + + it('uses the MicroVM boot-time policy', () => { + const provider = createMicrovmScaleDownProvider(); + const runner = { id: 'mvm-1', owner: 'Codertocat', type: 'Org' as const }; + + expect(provider.bootTimeExceeded(runner)).toBe(false); + expect(microvmBootTimeExceeded).toHaveBeenCalledWith(runner); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts new file mode 100644 index 0000000000..9ea9dc474a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -0,0 +1,28 @@ +import type { ScaleDownComputeProvider } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; + +export function createMicrovmScaleDownProvider(): Omit { + const imageArnByRunnerId = new Map(); + + async function list(environment: string, orphan?: boolean): Promise { + const runners = await listMicrovmRunners({ environment, orphan }); + for (const runner of runners) { + if (runner.imageArn) imageArnByRunnerId.set(runner.id, runner.imageArn); + } + return runners; + } + + function imageArnForRunner(id: string): string { + return imageArnByRunnerId.get(id) ?? loadMicrovmProviderConfig().imageIdentifier; + } + + return { + list, + bootTimeExceeded: microvmBootTimeExceeded, + markOrphan: async (id) => await tagMicrovm(imageArnForRunner(id), id, { 'ghr:orphan': 'true' }), + unmarkOrphan: async (id) => await untagMicrovm(imageArnForRunner(id), id, ['ghr:orphan']), + terminate: terminateMicrovm, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts new file mode 100644 index 0000000000..cfbbda3257 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -0,0 +1,114 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; +import { createMicrovmScaleUpProvider } from './scale-up'; + +vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn() })); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], +}; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-current', owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-new'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('createMicrovmScaleUpProvider', () => { + it('resolves supported resource override labels and registers them on the runner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.resolveLabelsForRunners([ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]), + ).resolves.toEqual({ + runnerLabels: [ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ], + state: { + overrides: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }, + }, + }); + }); + + it('rejects unsupported MicroVM override labels at the control-plane boundary', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect(provider.resolveLabelsForRunners(['ghr-microvm-memory:8192'])).rejects.toThrow( + "key 'memory' is not a supported MicroVM override", + ); + }); + + it('counts managed MicroVMs for the runner owner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.getCurrentRunners({ overrides: {} }, { runnerOwner: 'Codertocat', runnerType: 'Org' }), + ).resolves.toBe(1); + expect(listMicrovmRunners).toHaveBeenCalledWith({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }); + }); + + it('delegates runner creation to the shared MicroVM lifecycle', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.createRunners({ + githubRunnerConfig, + numberOfRunners: 1, + githubInstallationClient: githubClient, + state: { overrides: { imageVersion: '3.0' } }, + }), + ).resolves.toEqual({ instances: ['mvm-new'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + expect(createMicrovmRunners).toHaveBeenCalledWith( + githubRunnerConfig, + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + { imageVersion: '3.0' }, + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts new file mode 100644 index 0000000000..a3dcf1219e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts @@ -0,0 +1,77 @@ +import type { + CreateRunnerResult, + CreateScaleUpRunnersInput, + CreateStartRunnerConfig, + CurrentRunnersInput, + RunnerLabelResolution, + ScaleUpComputeProvider, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +interface MicrovmScaleUpState { + overrides: MicrovmDynamicLabelOverrides; +} + +async function resolveMicrovmLabelsForRunners( + messageLabels: string[], +): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const parsed = parseMicrovmDynamicLabels(trimmedLabels); + if (parsed.violations.length > 0) { + throw new Error( + `Invalid MicroVM dynamic labels: ${parsed.violations + .map((violation) => `${violation.label} (${violation.reason})`) + .join(', ')}`, + ); + } + + return { + runnerLabels: trimmedLabels.filter((label) => label.startsWith('ghr-')), + state: { overrides: parsed.overrides }, + }; +} + +async function getCurrentMicrovmRunners( + _state: MicrovmScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return ( + await listMicrovmRunners({ + environment: process.env.ENVIRONMENT, + runnerType, + runnerOwner, + }) + ).length; +} + +async function createMicrovmScaleUpRunners( + { + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, + }: CreateScaleUpRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + return await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'scale-up-lambda', + state.overrides, + ); +} + +export function createMicrovmScaleUpProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + resolveLabelsForRunners: resolveMicrovmLabelsForRunners, + getCurrentRunners: getCurrentMicrovmRunners, + createRunners: (input) => createMicrovmScaleUpRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts new file mode 100644 index 0000000000..6ea669a143 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { parseMicrovmDynamicLabels } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const internetEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS'; + +describe('parseMicrovmDynamicLabels', () => { + it('parses every supported RunMicrovm override', () => { + expect( + parseMicrovmDynamicLabels([ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]), + ).toEqual({ + overrides: { + egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], + imageIdentifier: imageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }, + violations: [], + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-egress-network-connectors:not-an-arn', + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn};${internetEgressConnectorArn}`, + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], + ['ghr-microvm-image-version:', "key 'image-version' requires a value"], + ['ghr-microvm-maximum-duration-in-seconds:0', 'maximum duration must be an integer between 1 and 28800'], + ['ghr-microvm-maximum-duration-in-seconds:28801', 'maximum duration must be an integer between 1 and 28800'], + ])('rejects invalid override %s', (label, reason) => { + const result = parseMicrovmDynamicLabels([label]); + + expect(result.overrides).toEqual({}); + expect(result.violations).toEqual([{ label, reason: expect.stringContaining(reason) }]); + }); + + it('ignores generic dynamic labels', () => { + expect(parseMicrovmDynamicLabels(['ghr-team:platform'])).toEqual({ overrides: {}, violations: [] }); + }); + + it('rejects more than ten egress network connectors', () => { + const labels = Array.from( + { length: 11 }, + (_, index) => + `ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:connector-${index}`, + ); + + const result = parseMicrovmDynamicLabels(labels); + + expect(result.overrides.egressNetworkConnectors).toHaveLength(10); + expect(result.violations).toEqual([ + { + label: labels[10], + reason: 'at most 10 egress network connector labels are supported', + }, + ]); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts new file mode 100644 index 0000000000..50c223cfd2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -0,0 +1,90 @@ +export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; + +const MAXIMUM_DURATION_IN_SECONDS = 28_800; +const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; +const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = + /^arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:(?:[0-9]{12}|aws):network-connector:[a-zA-Z0-9_-]+(?::[a-zA-Z0-9_-]+)?$/; + +export interface MicrovmDynamicLabelOverrides { + egressNetworkConnectors?: string[]; + imageIdentifier?: string; + imageVersion?: string; + maximumDurationInSeconds?: number; +} + +export interface MicrovmDynamicLabelViolation { + label: string; + reason: string; +} + +export function parseMicrovmDynamicLabels(labels: string[]): { + overrides: MicrovmDynamicLabelOverrides; + violations: MicrovmDynamicLabelViolation[]; +} { + const overrides: MicrovmDynamicLabelOverrides = {}; + const violations: MicrovmDynamicLabelViolation[] = []; + + for (const label of labels) { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) continue; + + const stripped = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? '' : stripped.slice(colonIndex + 1).trim(); + + if (!value) { + violations.push({ label, reason: `key '${key}' requires a value` }); + continue; + } + + switch (key) { + case 'egress-network-connectors': { + if (!MICROVM_NETWORK_CONNECTOR_ARN_PATTERN.test(value)) { + violations.push({ + label, + reason: `'${value}' is not a valid Lambda network connector ARN; specify one ARN per label`, + }); + break; + } + + const connectors = overrides.egressNetworkConnectors ?? []; + if (connectors.length >= MAXIMUM_EGRESS_NETWORK_CONNECTORS) { + violations.push({ + label, + reason: `at most ${MAXIMUM_EGRESS_NETWORK_CONNECTORS} egress network connector labels are supported`, + }); + } else { + overrides.egressNetworkConnectors = [...connectors, value]; + } + break; + } + case 'image-arn': + if (!MICROVM_IMAGE_ARN_PATTERN.test(value)) { + violations.push({ label, reason: `'${value}' is not a valid customer MicroVM image ARN` }); + } else { + overrides.imageIdentifier = value; + } + break; + case 'image-version': + overrides.imageVersion = value; + break; + case 'maximum-duration-in-seconds': { + const duration = Number(value); + if (!Number.isInteger(duration) || duration < 1 || duration > MAXIMUM_DURATION_IN_SECONDS) { + violations.push({ + label, + reason: `maximum duration must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, + }); + } else { + overrides.maximumDurationInSeconds = duration; + } + break; + } + default: + violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); + } + } + + return { overrides, violations }; +} diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index a6ebc12163..572b124092 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -11,7 +11,8 @@ "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", - "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts" + "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", "license": "MIT", diff --git a/lambdas/libs/compute-providers/providers.config.control-plane.ts b/lambdas/libs/compute-providers/providers.config.control-plane.ts index 55ebaca95e..45a584bc06 100644 --- a/lambdas/libs/compute-providers/providers.config.control-plane.ts +++ b/lambdas/libs/compute-providers/providers.config.control-plane.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/control-plane'; +import { provider as microvm } from './aws/microvm/control-plane'; import type { ControlPlaneProviderModule } from './contracts'; /** Provider plugins included in the control-plane bundle. */ -export const enabledControlPlaneProviders = [ec2] as const satisfies readonly ControlPlaneProviderModule[]; +export const enabledControlPlaneProviders = [ec2, microvm] as const satisfies readonly ControlPlaneProviderModule[]; From a8d59f685b7306b64088e4e3b9e9527addfd87d3 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 20:42:21 +0200 Subject: [PATCH 12/21] feat(compute-providers): add MicroVM webhook routing --- .../src/webhook/dynamic-labels-policy.ts | 61 ++++++++++ .../src/webhook/dynamic-labels.test.ts | 105 ++++++++++++++++++ .../aws/microvm/src/webhook/dynamic-labels.ts | 48 ++++++++ .../aws/microvm/webhook.test.ts | 36 ++++++ .../compute-providers/aws/microvm/webhook.ts | 16 +++ lambdas/libs/compute-providers/package.json | 1 + .../providers.config.webhook.ts | 3 +- 7 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/webhook.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/webhook.ts diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts new file mode 100644 index 0000000000..9785383727 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts @@ -0,0 +1,61 @@ +import type { AwsDynamicLabelsPolicy } from '../../../../contracts'; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${pattern}$`); +} + +function matchesAny(value: string, patterns: string[] | undefined): boolean { + if (!patterns || patterns.length === 0) return false; + return patterns.some((pattern) => globToRegExp(pattern).test(value)); +} + +function evaluateLabel(label: string, policy: AwsDynamicLabelsPolicy, labelPrefix: string): string | null { + const stripped = label.slice(labelPrefix.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? undefined : stripped.slice(colonIndex + 1); + + if (policy.blocked_keys?.includes(key)) { + return `key '${key}' is in blocked_keys`; + } + + const rule = policy.restricted_keys?.[key]; + if (!rule || value === undefined) return null; + + if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { + return `value '${value}' not in allowed list`; + } + if (rule.denied && matchesAny(value, rule.denied)) { + return `value '${value}' in denied list`; + } + if (rule.max !== undefined && rule.max !== null) { + const valueNumber = Number(value); + const maximum = Number(rule.max); + if (!Number.isFinite(valueNumber) || !Number.isFinite(maximum)) { + return `max set but value '${value}' or max '${rule.max}' is not numeric`; + } + if (valueNumber > maximum) { + return `value '${value}' exceeds max '${rule.max}'`; + } + } + + return null; +} + +export function violationsAgainstAwsDynamicLabelsPolicy( + labels: string[], + policy: AwsDynamicLabelsPolicy | null | undefined, + labelPrefix: string, +): { label: string; reason: string }[] { + if (!policy) return []; + + const violations: { label: string; reason: string }[] = []; + for (const label of labels) { + if (!label.startsWith(labelPrefix)) continue; + const reason = evaluateLabel(label, policy, labelPrefix); + if (reason) violations.push({ label, reason }); + } + return violations; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts new file mode 100644 index 0000000000..6b157309af --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../../../contracts'; +import { microvmDynamicLabelProvider } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + +describe('microvmDynamicLabelProvider', () => { + it('accepts supported MicroVM overrides', () => { + const queue = microvmQueue(); + const dynamicLabels = [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]; + + expect(selectQueue(queue, dynamicLabels)).toEqual({ + queue, + labels: ['self-hosted', 'linux', ...dynamicLabels], + }); + }); + + it('rejects dynamic labels when the queue disables them', () => { + const queue = microvmQueue(); + queue.matcherConfig.enableDynamicLabels = false; + + expect(selectQueue(queue, ['ghr-microvm-image-version:3.0'])).toBeUndefined(); + }); + + it('rejects unsupported MicroVM resource overrides', () => { + expect(selectQueue(microvmQueue(), ['ghr-microvm-memory:8192'])).toBeUndefined(); + }); + + it('enforces the AWS dynamic-label policy', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { 'maximum-duration-in-seconds': { max: 3600 } }, + }; + + expect(selectQueue(queue, ['ghr-microvm-maximum-duration-in-seconds:7200'])).toBeUndefined(); + }); + + it('applies allowed patterns to the complete image ARN', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-arn': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-*'], + }, + }, + }; + + expect( + selectQueue(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large']), + ).toBeDefined(); + expect( + selectQueue(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toBeUndefined(); + }); + + it('applies the policy to each egress connector label', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-*'], + }, + }, + }; + + expect( + selectQueue(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', + ]), + ).toBeDefined(); + expect( + selectQueue(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', + ]), + ).toBeUndefined(); + }); +}); + +function selectQueue(queue: RunnerMatcherConfig, sanitizedGhrLabels: string[]) { + return microvmDynamicLabelProvider.selectQueue({ + queue, + nonGhrLabels: ['self-hosted', 'linux'], + sanitizedGhrLabels, + }); +} + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux', 'arm64', 'microvm']], + exactMatch: false, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..36eb3e7670 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -0,0 +1,48 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { DynamicLabelDispatchTarget, DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; +import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { violationsAgainstAwsDynamicLabelsPolicy } from './dynamic-labels-policy'; + +const logger = createChildLogger('handler'); + +export function selectMicrovmDynamicLabelQueue( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], +): DynamicLabelDispatchTarget | undefined { + for (const queue of matches) { + if (!queue.matcherConfig.enableDynamicLabels) { + logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); + continue; + } + + const parsedLabels = parseMicrovmDynamicLabels(sanitizedGhrLabels); + const policyViolations = violationsAgainstAwsDynamicLabelsPolicy( + sanitizedGhrLabels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ); + const violations = [...parsedLabels.violations, ...policyViolations]; + + if (violations.length === 0) { + return { + queue, + labels: [...nonGhrLabels, ...sanitizedGhrLabels], + }; + } + + for (const violation of violations) { + logger.warn( + `Queue ${queue.id}: dynamic label '${violation.label}' is not accepted (${violation.reason}); trying next match`, + ); + } + } + + return undefined; +} + +export const microvmDynamicLabelProvider: DynamicLabelProvider = { + selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => + selectMicrovmDynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), +}; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts new file mode 100644 index 0000000000..bdc6d2918c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../contracts'; +import { provider } from './webhook'; + +describe('MicroVM webhook provider contract', () => { + it('exposes MicroVM dynamic-label selection', () => { + const plugin = provider.createPlugin(); + const queue = microvmQueue(); + + expect(plugin.type).toBe('microvm'); + expect( + plugin.capabilities.dynamicLabels.selectQueue({ + queue, + nonGhrLabels: ['self-hosted', 'linux'], + sanitizedGhrLabels: ['ghr-microvm-image-version:3.0'], + }), + ).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-microvm-image-version:3.0'], + }); + }); +}); + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.ts new file mode 100644 index 0000000000..48d603e476 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.ts @@ -0,0 +1,16 @@ +import type { ComputeProviderPlugin } from '../../core'; + +import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; +import { microvmDynamicLabelProvider } from './src/webhook/dynamic-labels'; + +export function createMicrovmWebhookPlugin(): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { dynamicLabels: microvmDynamicLabelProvider }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmWebhookPlugin, +} satisfies WebhookProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index 572b124092..cd03f897f0 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -12,6 +12,7 @@ "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/webhook": "./aws/microvm/webhook.ts", "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", diff --git a/lambdas/libs/compute-providers/providers.config.webhook.ts b/lambdas/libs/compute-providers/providers.config.webhook.ts index 19c92734da..a4aec0853a 100644 --- a/lambdas/libs/compute-providers/providers.config.webhook.ts +++ b/lambdas/libs/compute-providers/providers.config.webhook.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/webhook'; +import { provider as microvm } from './aws/microvm/webhook'; import type { WebhookProviderModule } from './contracts'; /** Provider plugins included in the webhook bundle. */ -export const enabledWebhookProviders = [ec2] as const satisfies readonly WebhookProviderModule[]; +export const enabledWebhookProviders = [ec2, microvm] as const satisfies readonly WebhookProviderModule[]; From 979c88d7262bc09d00ff812e973bab59696f24d5 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 6 Aug 2026 20:42:41 +0200 Subject: [PATCH 13/21] docs(compute-providers): document Lambda MicroVM provider --- .../compute-providers/aws/microvm/README.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 lambdas/libs/compute-providers/aws/microvm/README.md diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..e729bd4383 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -0,0 +1,70 @@ +# Lambda MicroVM compute provider + +This provider manages a compatible AWS Lambda MicroVM image through the control-plane Lambda. It currently supports ephemeral JIT runners only. + +The MicroVM image `/run` hook receives this `runHookPayload`: + +```json +{ + "version": 1, + "runnerConfigSsmPath": "/github-action-runners/example/token" +} +``` + +Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and terminate the MicroVM after the job completes. + +The control-plane Lambda requires these provider environment variables: + +- `MICROVM_IMAGE_ARN` +- `MICROVM_EXECUTION_ROLE_ARN` +- `MICROVM_IMAGE_VERSION` (optional) +- `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_MAXIMUM_DURATION_IN_SECONDS` (optional, defaults to 3600) +- `MICROVM_LOG_GROUP` (optional) + +## Dynamic labels + +When a runner matcher enables dynamic labels, workflow jobs can override the +following `RunMicrovm` inputs: + +| Label | Override | +| --------------------------------------------------- | ---------------------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | +| `ghr-microvm-maximum-duration-in-seconds:` | Maximum lifetime from 1 through 28,800 seconds | + +Repeat `ghr-microvm-egress-network-connectors:` to attach multiple +connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These +labels replace the compute provider's configured +`MICROVM_EGRESS_NETWORK_CONNECTORS` value for that job. + +Lambda MicroVM does not expose CPU or memory as `RunMicrovm` inputs. Select an +image and version with the required resources instead. Labels such as +`ghr-microvm-memory` are rejected. + +Execution roles, ingress network connectors, logging, idle policy, run hook +payloads, and client tokens remain deployment-controlled. Egress connector +overrides change the runner's network boundary and should be restricted to +approved connector ARNs with `awsDynamicLabelsPolicy`. + +Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from +workflow jobs. The MicroVM policy keys are `egress-network-connectors`, +`image-arn`, `image-version`, and `maximum-duration-in-seconds`. For example: + +```json +{ + "restricted_keys": { + "egress-network-connectors": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-*"] + }, + "image-arn": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] + }, + "maximum-duration-in-seconds": { + "max": 3600 + } + } +} +``` From f10a155e86f8658c7560a57704aebd858d3cc282 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 20:11:04 +0200 Subject: [PATCH 14/21] fix(compute-providers): replace unsupported MicroVM tags --- lambdas/libs/aws-ssm-util/src/index.test.ts | 77 +++- lambdas/libs/aws-ssm-util/src/index.ts | 54 ++- .../compute-providers/aws/microvm/README.md | 42 ++- .../microvm/src/control-plane/config.test.ts | 14 + .../aws/microvm/src/control-plane/config.ts | 10 + .../src/control-plane/microvms.test.ts | 269 ++++++++------ .../aws/microvm/src/control-plane/microvms.ts | 136 +++---- .../src/control-plane/runner-config.test.ts | 39 +- .../src/control-plane/runner-config.ts | 12 +- .../src/control-plane/runner-metadata.test.ts | 253 +++++++++++++ .../src/control-plane/runner-metadata.ts | 348 ++++++++++++++++++ .../src/control-plane/scale-down.test.ts | 48 +-- .../microvm/src/control-plane/scale-down.ts | 21 +- .../aws/microvm/src/environment.d.ts | 1 + .../src/webhook/dynamic-labels-policy.ts | 61 --- .../src/webhook/dynamic-labels.test.ts | 80 ++-- .../aws/microvm/src/webhook/dynamic-labels.ts | 62 ++-- .../aws/microvm/webhook.test.ts | 57 ++- 18 files changed, 1202 insertions(+), 382 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts delete mode 100644 lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index 8a1d8d3864..ad68c12279 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -1,6 +1,8 @@ import { + DeleteParameterCommand, GetParameterCommand, GetParameterCommandOutput, + GetParametersByPathCommand, GetParametersCommand, PutParameterCommand, PutParameterCommandOutput, @@ -10,7 +12,16 @@ import 'aws-sdk-client-mock-jest/vitest'; import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; -import { getParameter, getParameters, putParameter, resetSSMClient, ssmClient, SSM_ADVANCED_TIER_THRESHOLD } from '.'; +import { + deleteParameter, + getParameter, + getParameters, + getParametersByPath, + putParameter, + resetSSMClient, + ssmClient, + SSM_ADVANCED_TIER_THRESHOLD, +} from '.'; import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockSSMClient = mockClient(SSMClient); @@ -104,6 +115,30 @@ describe('Test getParameter and putParameter', () => { }); }); + it('overwrites a parameter only when explicitly requested', async () => { + mockSSMClient.on(PutParameterCommand).resolves({}); + + await putParameter('testParam', 'updated', false, { overwrite: true }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: 'testParam', + Value: 'updated', + Type: 'String', + Overwrite: true, + }); + }); + + it('rejects tags when overwriting an existing parameter', async () => { + mockSSMClient.resetHistory(); + await expect( + putParameter('testParam', 'updated', false, { + overwrite: true, + tags: [{ Key: 'owner', Value: 'runner' }], + } as never), + ).rejects.toThrow('tags cannot be supplied when overwriting'); + expect(mockSSMClient).not.toHaveReceivedCommand(PutParameterCommand); + }); + it('Puts parameters as SecureString', async () => { // Arrange const parameterValue = 'test'; @@ -256,6 +291,46 @@ describe('Test getParameters (batch)', () => { }); }); +describe('Test direct parameter path operations', () => { + beforeEach(() => { + mockSSMClient.reset(); + }); + + it('paginates direct, non-secret children of a parameter path', async () => { + mockSSMClient + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: undefined, + }) + .resolves({ Parameters: [{ Name: '/metadata/one', Value: '1' }], NextToken: 'page-2' }) + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: 'page-2', + }) + .resolves({ Parameters: [{ Name: '/metadata/two', Value: '2' }] }); + + await expect(getParametersByPath('/metadata')).resolves.toEqual( + new Map([ + ['/metadata/one', '1'], + ['/metadata/two', '2'], + ]), + ); + expect(mockSSMClient).toHaveReceivedCommandTimes(GetParametersByPathCommand, 2); + }); + + it('deletes an exact parameter name', async () => { + mockSSMClient.on(DeleteParameterCommand).resolves({}); + + await deleteParameter('/metadata/one'); + + expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: '/metadata/one' }); + }); +}); + describe('SSM client configuration', () => { it('configures adaptive retry with a raised attempt cap', async () => { const config = ssmClient().config; diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 71b33cbf41..9fef6c7b97 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -1,4 +1,11 @@ -import { GetParametersCommand, PutParameterCommand, SSMClient, Tag } from '@aws-sdk/client-ssm'; +import { + DeleteParameterCommand, + GetParametersByPathCommand, + GetParametersCommand, + PutParameterCommand, + SSMClient, + Tag, +} from '@aws-sdk/client-ssm'; import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm'; @@ -103,14 +110,56 @@ export async function getParameters(parameter_names: string[]): Promise> { + const result = new Map(); + let nextToken: string | undefined; + + do { + const response = await ssmClient().send( + new GetParametersByPathCommand({ + Path: parameter_path, + Recursive: false, + WithDecryption: false, + NextToken: nextToken, + }), + ); + + for (const parameter of response.Parameters ?? []) { + if (parameter.Name && parameter.Value) { + result.set(parameter.Name, parameter.Value); + } + } + nextToken = response.NextToken; + } while (nextToken); + + return result; +} + +export async function deleteParameter(parameter_name: string): Promise { + await ssmClient().send(new DeleteParameterCommand({ Name: parameter_name })); +} + export const SSM_ADVANCED_TIER_THRESHOLD = 4000; +type PutParameterOptions = { overwrite: true; tags?: never } | { overwrite?: false | undefined; tags?: Tag[] }; + export async function putParameter( parameter_name: string, parameter_value: string, secure: boolean, - options: { tags?: Tag[] } = {}, + options: PutParameterOptions = {}, ): Promise { + if (options.overwrite && options.tags !== undefined) { + throw new Error('SSM parameter tags cannot be supplied when overwriting an existing parameter'); + } + const client = ssmClient(); // Determine tier based on parameter_value size @@ -121,6 +170,7 @@ export async function putParameter( Name: parameter_name, Value: parameter_value, Type: secure ? 'SecureString' : 'String', + Overwrite: options.overwrite, Tags: options.tags, Tier: valueSizeBytes >= SSM_ADVANCED_TIER_THRESHOLD ? 'Advanced' : 'Standard', }), diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index e729bd4383..71e53f9f7b 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -13,6 +13,15 @@ The MicroVM image `/run` hook receives this `runHookPayload`: Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and terminate the MicroVM after the job completes. +Runner ownership and lifecycle state are stored separately as non-secret `String` +parameters under `/`. The immutable base +record and independent state parameters prevent concurrent GitHub ID, orphan, +and cleanup updates from overwriting one another. Deleting the JIT SecureString +does not delete this metadata. Use a dedicated metadata prefix that does not +overlap the JIT path, and do not grant the MicroVM execution role access to it. +The control plane retries pending cleanup, removes metadata after termination, +and reconciles expired records during inventory. + The control-plane Lambda requires these provider environment variables: - `MICROVM_IMAGE_ARN` @@ -21,8 +30,31 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_MAXIMUM_DURATION_IN_SECONDS` (optional, defaults to 3600) +- `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) - `MICROVM_LOG_GROUP` (optional) +The control-plane role requires `ssm:GetParametersByPath`, `ssm:PutParameter`, +and `ssm:DeleteParameter` on the dedicated metadata prefix, plus +`lambda:ListMicrovms`, `lambda:RunMicrovm`, and `lambda:TerminateMicrovm` for +inventory and lifecycle reconciliation. Restrict `lambda:RunMicrovm` and +`lambda:TerminateMicrovm` to approved image resources; `lambda:ListMicrovms` +does not support resource-level permissions. + +The MicroVM execution role must trust `lambda.amazonaws.com` for both +`sts:AssumeRole` and `sts:TagSession`. Restrict `iam:PassRole` to that exact role +with `iam:PassedToService=lambda.amazonaws.com`. Egress connectors also require +`lambda:PassNetworkConnector`; because that action does not currently support +resource-level permissions, enforce the connector boundary with the explicit +dynamic-label allowlist described below. + +All MicroVMs using one execution role and JIT prefix share a trust boundary. +Grant that role only `ssm:GetParameter` and `ssm:DeleteParameter` on the JIT +prefix; do not grant parameter-listing APIs or access to the metadata prefix. +The `MicrovmId` tag on each JIT parameter supports operations but is not a +documented binding to the calling MicroVM's session identity. Only allow trusted +images and workloads within a shared role, or isolate trust domains with +separate roles, prefixes, and provider deployments. + ## Dynamic labels When a runner matcher enables dynamic labels, workflow jobs can override the @@ -45,9 +77,10 @@ image and version with the required resources instead. Labels such as `ghr-microvm-memory` are rejected. Execution roles, ingress network connectors, logging, idle policy, run hook -payloads, and client tokens remain deployment-controlled. Egress connector -overrides change the runner's network boundary and should be restricted to -approved connector ARNs with `awsDynamicLabelsPolicy`. +payloads, and client tokens remain deployment-controlled. Image ARN, image +version, and egress connector overrides change executable code or the network +boundary, so they are rejected unless `awsDynamicLabelsPolicy` supplies an +explicit `allowed` list for the corresponding key. Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from workflow jobs. The MicroVM policy keys are `egress-network-connectors`, @@ -62,6 +95,9 @@ workflow jobs. The MicroVM policy keys are `egress-network-connectors`, "image-arn": { "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] }, + "image-version": { + "allowed": ["3.*"] + }, "maximum-duration-in-seconds": { "max": 3600 } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index ce692a1ab4..1cc240a9f7 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -8,6 +8,7 @@ beforeEach(() => { process.env = { ...cleanEnv }; process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; @@ -24,6 +25,7 @@ describe('loadMicrovmProviderConfig', () => { ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, maximumDurationInSeconds: 3600, + metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', logging: undefined, }); }); @@ -47,6 +49,7 @@ describe('loadMicrovmProviderConfig', () => { it.each([ ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], ])('requires %s', (environmentVariable, expectedName) => { delete process.env[environmentVariable]; @@ -68,4 +71,15 @@ describe('loadMicrovmProviderConfig', () => { expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); }); + + it.each(['metadata', '/', '/metadata//nested', '/metadata/has space'])( + 'rejects malformed metadata SSM path %s', + (metadataPath) => { + process.env.MICROVM_METADATA_SSM_PATH = metadataPath; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path', + ); + }, + ); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index 7c0a7662b9..ddd7891362 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -11,6 +11,7 @@ export interface MicrovmProviderConfig { ingressNetworkConnectors?: string[]; logging?: Logging; maximumDurationInSeconds: number; + metadataSsmPath: string; } function requiredEnvironmentValue(name: string, value: string | undefined): string { @@ -26,6 +27,14 @@ function optionalEnvironmentValue(value: string | undefined): string | undefined return trimmed ? trimmed : undefined; } +function parseMetadataSsmPath(value: string | undefined): string { + const path = requiredEnvironmentValue('MICROVM_METADATA_SSM_PATH', value).replace(/\/+$/, ''); + if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//')) { + throw new Error('MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path'); + } + return path; +} + function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { const configuredValue = optionalEnvironmentValue(value); if (!configuredValue) return undefined; @@ -83,6 +92,7 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, ), maximumDurationInSeconds: parseMaximumDuration(process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS), + metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index 7d7199a3cc..cd4fe86250 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -1,11 +1,8 @@ import { LambdaMicrovmsClient, ListMicrovmsCommand, - ListTagsCommand, RunMicrovmCommand, - TagResourceCommand, TerminateMicrovmCommand, - UntagResourceCommand, } from '@aws-sdk/client-lambda-microvms'; import { mockClient } from 'aws-sdk-client-mock'; import 'aws-sdk-client-mock-jest/vitest'; @@ -15,49 +12,69 @@ import type { MicrovmProviderConfig } from './config'; import { isRetryableMicrovmError, listMicrovmRunners, - microvmArn, microvmBootTimeExceeded, runMicrovmRunner, - tagMicrovm, terminateMicrovm, - untagMicrovm, } from './microvms'; +import { + createMicrovmRunnerMetadata, + deleteMicrovmRunnerMetadata, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('./runner-metadata', () => ({ + createMicrovmRunnerMetadata: vi.fn(), + deleteMicrovmRunnerMetadata: vi.fn(), + listMicrovmRunnerMetadata: vi.fn(), + markMicrovmCleanupPending: vi.fn(), +})); const mockMicrovmClient = mockClient(LambdaMicrovmsClient); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const config: MicrovmProviderConfig = { imageIdentifier: imageArn, imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', egressNetworkConnectors: ['arn:egress'], maximumDurationInSeconds: 1200, + metadataSsmPath, logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-managed', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.0', + createdAt: '2026-08-06T10:00:00.000Z', + expiresAt: '2026-08-06T11:00:00.000Z', + ...overrides, + }; +} + beforeEach(() => { mockMicrovmClient.reset(); + vi.clearAllMocks(); vi.useRealTimers(); process.env.AWS_REGION = 'eu-west-1'; process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; -}); - -describe('microvmArn', () => { - it('derives the MicroVM resource ARN from its image ARN', () => { - expect(microvmArn(imageArn, 'mvm-123')).toBe('arn:aws:lambda:eu-west-1:123456789012:microvm:mvm-123'); - expect(microvmArn(imageArn.replace('arn:aws:', 'arn:aws-us-gov:'), 'mvm-456')).toContain('arn:aws-us-gov:lambda:'); - }); - - it('rejects image names that cannot identify a customer MicroVM resource', () => { - expect(() => microvmArn('runner', 'mvm-123')).toThrow( - 'MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN', - ); - }); + vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(); + vi.mocked(deleteMicrovmRunnerMetadata).mockResolvedValue(); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ cleanupMicrovmIds: [], metadataById: new Map() }); + vi.mocked(markMicrovmCleanupPending).mockResolvedValue(); }); describe('runMicrovmRunner', () => { - it('launches and tags a managed runner', async () => { - mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123' }); - mockMicrovmClient.on(TagResourceCommand).resolves({}); + it('launches a runner and records durable ownership metadata', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn }); await expect( runMicrovmRunner({ @@ -80,15 +97,15 @@ describe('runMicrovmRunner', () => { runHookPayload: '{"version":1}', clientToken: expect.any(String), }); - expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { - Resource: microvmArn(imageArn, 'mvm-123'), - Tags: { - 'ghr:Application': 'github-action-runner', - 'ghr:created_by': 'scale-up-lambda', - 'ghr:environment': 'unit-test', - 'ghr:Owner': 'Codertocat', - 'ghr:Type': 'Org', - }, + expect(createMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, { + microvmId: 'mvm-123', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 1200, }); }); @@ -107,11 +124,10 @@ describe('runMicrovmRunner', () => { ).rejects.toThrow('RunMicrovm returned no microvmId'); }); - it('terminates a new runner when required tags cannot be applied', async () => { - const tagError = new Error('tag failed'); - mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); - mockMicrovmClient.on(TagResourceCommand).rejects(tagError); + it('terminates a new runner when required metadata cannot be recorded', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); await expect( runMicrovmRunner({ @@ -122,17 +138,17 @@ describe('runMicrovmRunner', () => { runnerType: 'Org', source: 'scale-up-lambda', }), - ).rejects.toThrow('tag failed'); + ).rejects.toThrow('metadata failed'); expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { - microvmIdentifier: 'mvm-untagged', + microvmIdentifier: 'mvm-untracked', }); }); - it('preserves the tag error when cleanup also fails', async () => { - mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); - mockMicrovmClient.on(TagResourceCommand).rejects(new Error('tag failed')); + it('preserves the metadata error when termination also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); await expect( runMicrovmRunner({ @@ -143,12 +159,13 @@ describe('runMicrovmRunner', () => { runnerType: 'Org', source: 'scale-up-lambda', }), - ).rejects.toThrow('tag failed'); + ).rejects.toThrow('metadata failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-untracked'); }); }); describe('listMicrovmRunners', () => { - it('paginates active MicroVMs and filters them by management tags', async () => { + it('paginates active MicroVMs and filters them by durable metadata', async () => { const startedAt = new Date('2026-08-06T10:00:00.000Z'); mockMicrovmClient .on(ListMicrovmsCommand) @@ -162,26 +179,23 @@ describe('listMicrovmRunners', () => { .resolvesOnce({ items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], }); - mockMicrovmClient - .on(ListTagsCommand) - .resolvesOnce({ - Tags: { - 'ghr:Application': 'github-action-runner', - 'ghr:environment': 'unit-test', - 'ghr:Owner': 'Codertocat', - 'ghr:Type': 'Org', - 'ghr:github_runner_id': '42', - 'ghr:bypass-removal': 'true', - }, - }) - .resolvesOnce({ Tags: { 'ghr:Application': 'another-application' } }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-managed', metadata({ githubRunnerId: '42', bypassRemoval: true })], + ['mvm-other', metadata({ microvmId: 'mvm-other', runnerOwner: 'Other' })], + ]), + }); await expect( - listMicrovmRunners({ - environment: 'unit-test', - runnerOwner: 'Codertocat', - runnerType: 'Org', - }), + listMicrovmRunners( + { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }, + metadataSsmPath, + ), ).resolves.toEqual([ { id: 'mvm-managed', @@ -200,9 +214,17 @@ describe('listMicrovmRunners', () => { maxResults: 50, nextToken: 'page-2', }); + expect(listMicrovmRunnerMetadata).toHaveBeenCalledWith( + metadataSsmPath, + new Map([ + ['mvm-managed', 'RUNNING'], + ['mvm-terminated', 'TERMINATED'], + ['mvm-other', 'PENDING'], + ]), + ); }); - it('applies environment, owner, type, and orphan filters after loading tags', async () => { + it('applies environment, owner, type, and orphan filters after loading metadata', async () => { mockMicrovmClient.on(ListMicrovmsCommand).resolves({ items: [ { @@ -214,62 +236,98 @@ describe('listMicrovmRunners', () => { }, ], }); - mockMicrovmClient.on(ListTagsCommand).resolves({ - Tags: { - 'ghr:Application': 'github-action-runner', - 'ghr:environment': 'other', - 'ghr:Owner': 'Other', - 'ghr:Type': 'Repo', - }, + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + [ + 'mvm-filtered', + metadata({ microvmId: 'mvm-filtered', environment: 'other', runnerOwner: 'Other', runnerType: 'Repo' }), + ], + ]), }); - await expect(listMicrovmRunners({ environment: 'unit-test' })).resolves.toEqual([]); - await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' })).resolves.toEqual([]); - await expect(listMicrovmRunners({ runnerType: 'Org' })).resolves.toEqual([]); - await expect(listMicrovmRunners({ orphan: true })).resolves.toEqual([]); + await expect(listMicrovmRunners({ environment: 'unit-test' }, metadataSsmPath)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, metadataSsmPath)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' }, metadataSsmPath)).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true }, metadataSsmPath)).resolves.toEqual([]); }); - it('skips a MicroVM that terminates before its tags can be read', async () => { - const resourceNotFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + it('fails closed for an image mismatch while ignoring unowned MicroVMs', async () => { mockMicrovmClient.on(ListMicrovmsCommand).resolves({ - items: [{ microvmId: 'mvm-gone', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + items: [ + { microvmId: 'mvm-missing', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-mismatch', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-mismatch', metadata({ microvmId: 'mvm-mismatch', imageArn: imageArn.replace(':runner', ':other') })], + ]), }); - mockMicrovmClient.on(ListTagsCommand).rejects(resourceNotFound); - await expect(listMicrovmRunners()).resolves.toEqual([]); + await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('does not match its metadata'); + }); + + it('attempts every pending cleanup and fails inventory closed when a retry fails', async () => { + const cleanupFailure = new Error('cleanup failed'); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-first', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-second', imageArn, imageVersion: '3.0', state: 'PENDING' }, + ], + }); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-first' }).rejects(cleanupFailure); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-second' }).resolves({}); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: ['mvm-first', 'mvm-second'], + metadataById: new Map(), + }); + + await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('cleanup failed'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-first', + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-second', + }); + expect(markMicrovmCleanupPending).toHaveBeenCalledTimes(2); }); - it('surfaces unexpected tag lookup failures', async () => { + it('surfaces metadata lookup failures instead of reporting zero runners', async () => { mockMicrovmClient.on(ListMicrovmsCommand).resolves({ - items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', state: 'RUNNING' }], }); - mockMicrovmClient.on(ListTagsCommand).rejects(new Error('list tags failed')); + vi.mocked(listMicrovmRunnerMetadata).mockRejectedValue(new Error('AccessDenied')); - await expect(listMicrovmRunners()).rejects.toThrow('list tags failed'); + await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('AccessDenied'); }); }); describe('MicroVM lifecycle helpers', () => { - it('tags, untags, and terminates a MicroVM', async () => { - mockMicrovmClient.on(TagResourceCommand).resolves({}); - mockMicrovmClient.on(UntagResourceCommand).resolves({}); + it('retains metadata until inventory observes a terminated MicroVM', async () => { mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); - await tagMicrovm(imageArn, 'mvm-123', { key: 'value' }); - await untagMicrovm(imageArn, 'mvm-123', ['key']); - await terminateMicrovm('mvm-123'); + await terminateMicrovm('mvm-123', metadataSsmPath); - expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { - Resource: microvmArn(imageArn, 'mvm-123'), - Tags: { key: 'value' }, - }); - expect(mockMicrovmClient).toHaveReceivedCommandWith(UntagResourceCommand, { - Resource: microvmArn(imageArn, 'mvm-123'), - TagKeys: ['key'], - }); - expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { - microvmIdentifier: 'mvm-123', - }); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + expect(deleteMicrovmRunnerMetadata).not.toHaveBeenCalled(); + }); + + it('treats an already terminated MicroVM as successful cleanup', async () => { + const notFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(notFound); + + await expect(terminateMicrovm('mvm-gone', metadataSsmPath)).resolves.toBeUndefined(); + expect(deleteMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + }); + + it('retains metadata and marks cleanup pending when termination fails', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect(terminateMicrovm('mvm-123', metadataSsmPath)).rejects.toThrow('terminate failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + expect(deleteMicrovmRunnerMetadata).not.toHaveBeenCalled(); }); it('evaluates the configured boot window', () => { @@ -283,12 +341,15 @@ describe('MicroVM lifecycle helpers', () => { }); describe('isRetryableMicrovmError', () => { - it.each(['ConflictException', 'InternalServerException', 'ServiceQuotaExceededException', 'ThrottlingException'])( - 'classifies %s as retryable', - (name) => { - expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); - }, - ); + it.each([ + 'ConflictException', + 'InternalServerException', + 'ServiceQuotaExceededException', + 'ThrottlingException', + 'TooManyUpdates', + ])('classifies %s as retryable', (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }); it('classifies server, throttling, network, and nested failures as retryable', () => { expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index edc4a3775b..3769d487c7 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -4,21 +4,22 @@ import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws- import { LambdaMicrovmsClient, ListMicrovmsCommand, - ListTagsCommand, RunMicrovmCommand, - TagResourceCommand, TerminateMicrovmCommand, - UntagResourceCommand, } from '@aws-sdk/client-lambda-microvms'; import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; import type { LambdaRunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; -import type { MicrovmProviderConfig } from './config'; +import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; +import { + createMicrovmRunnerMetadata, + deleteMicrovmRunnerMetadata, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, +} from './runner-metadata'; const logger = createChildLogger('microvm-runners'); -const APPLICATION_TAG = 'ghr:Application'; -const APPLICATION_TAG_VALUE = 'github-action-runner'; const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); export interface MicrovmRunnerInfo extends RunnerInfo { @@ -52,6 +53,7 @@ const RETRYABLE_ERROR_NAMES = new Set([ 'ServiceQuotaExceededException', 'Throttling', 'ThrottlingException', + 'TooManyUpdates', 'TooManyRequestsException', ]); @@ -68,16 +70,6 @@ function microvmClient(): LambdaMicrovmsClient { return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); } -export function microvmArn(imageArn: string, microvmId: string): string { - const match = /^arn:([^:]+):lambda:([^:]+):([0-9]{12}):microvm-image:.+$/.exec(imageArn); - if (!match) { - throw new Error(`MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN: ${imageArn}`); - } - - const [, partition, region, accountId] = match; - return `arn:${partition}:lambda:${region}:${accountId}:microvm:${microvmId}`; -} - export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { const commandInput: RunMicrovmCommandInput = { imageIdentifier: input.config.imageIdentifier, @@ -103,17 +95,22 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { - logger.error(`Failed to terminate untagged MicroVM runner '${response.microvmId}'`, { + logger.error(`Failed to record metadata for new MicroVM runner '${response.microvmId}', terminating it`, { + error, + }); + await terminateMicrovm(response.microvmId, input.config.metadataSsmPath).catch((terminationError) => { + logger.error(`Failed to terminate untracked MicroVM runner '${response.microvmId}'`, { error: terminationError, }); }); @@ -123,7 +120,10 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { +export async function listMicrovmRunners( + filters: ListRunnerFilters = {}, + metadataSsmPath = loadMicrovmProviderConfig().metadataSsmPath, +): Promise { const client = microvmClient(); const items: MicrovmItem[] = []; let nextToken: string | undefined; @@ -139,34 +139,50 @@ export async function listMicrovmRunners(filters: ListRunnerFilters = {}): Promi nextToken = response.nextToken; } while (nextToken); - const runners: MicrovmRunnerInfo[] = []; - for (const item of items) { - if (!item.microvmId || !item.imageArn || !item.state || !ACTIVE_STATES.has(item.state)) continue; + const activeItems = items.filter( + (item): item is MicrovmItem & { imageArn: string; microvmId: string; state: MicrovmState } => + Boolean(item.microvmId && item.imageArn && item.state && ACTIVE_STATES.has(item.state)), + ); + const microvmStates = new Map( + items.flatMap((item) => (item.microvmId && item.state ? [[item.microvmId, item.state] as const] : [])), + ); + const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(metadataSsmPath, microvmStates); - let tags: Record; + let cleanupError: unknown; + for (const microvmId of cleanupMicrovmIds) { + logger.warn(`Retrying cleanup of MicroVM runner '${microvmId}'`); try { - tags = - (await client.send(new ListTagsCommand({ Resource: microvmArn(item.imageArn, item.microvmId) }))).Tags ?? {}; + await terminateMicrovm(microvmId, metadataSsmPath); } catch (error) { - if (error instanceof Error && error.name === 'ResourceNotFoundException') continue; - throw error; + cleanupError ??= error; + logger.error(`Failed to retry cleanup of MicroVM runner '${microvmId}'`, { error }); } + } + if (cleanupError !== undefined) throw cleanupError; - if (tags[APPLICATION_TAG] !== APPLICATION_TAG_VALUE) continue; - if (filters.environment !== undefined && tags['ghr:environment'] !== filters.environment) continue; - if (filters.runnerType !== undefined && tags['ghr:Type'] !== filters.runnerType) continue; - if (filters.runnerOwner !== undefined && tags['ghr:Owner'] !== filters.runnerOwner) continue; - if (filters.orphan && tags['ghr:orphan'] !== 'true') continue; + const runners: MicrovmRunnerInfo[] = []; + for (const item of activeItems) { + const metadata = metadataById.get(item.microvmId); + if (!metadata) continue; + if (metadata.imageArn !== item.imageArn) { + throw new Error(`Active MicroVM runner '${item.microvmId}' has an image that does not match its metadata`); + } + + const orphan = Boolean(metadata.orphan); + if (filters.environment !== undefined && metadata.environment !== filters.environment) continue; + if (filters.runnerType !== undefined && metadata.runnerType !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && metadata.runnerOwner !== filters.runnerOwner) continue; + if (filters.orphan && !orphan) continue; runners.push({ id: item.microvmId, imageArn: item.imageArn, launchTime: item.startedAt, - owner: tags['ghr:Owner'], - type: tags['ghr:Type'] as RunnerInfo['type'], - orphan: tags['ghr:orphan'] === 'true', - githubRunnerId: tags['ghr:github_runner_id'], - bypassRemoval: tags['ghr:bypass-removal'] === 'true', + owner: metadata.runnerOwner, + type: metadata.runnerType, + orphan, + githubRunnerId: metadata.githubRunnerId, + bypassRemoval: metadata.bypassRemoval ?? false, state: item.state, }); } @@ -174,26 +190,22 @@ export async function listMicrovmRunners(filters: ListRunnerFilters = {}): Promi return runners; } -export async function tagMicrovm(imageArn: string, microvmId: string, tags: Record): Promise { - await microvmClient().send( - new TagResourceCommand({ - Resource: microvmArn(imageArn, microvmId), - Tags: tags, - }), - ); -} +export async function terminateMicrovm(microvmId: string, metadataSsmPath: string): Promise { + try { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') { + await deleteMicrovmRunnerMetadata(metadataSsmPath, microvmId); + return; + } -export async function untagMicrovm(imageArn: string, microvmId: string, tagKeys: string[]): Promise { - await microvmClient().send( - new UntagResourceCommand({ - Resource: microvmArn(imageArn, microvmId), - TagKeys: tagKeys, - }), - ); -} + await markMicrovmCleanupPending(metadataSsmPath, microvmId).catch((metadataError) => { + logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error: metadataError }); + }); + throw error; + } -export async function terminateMicrovm(microvmId: string): Promise { - await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); + await markMicrovmCleanupPending(metadataSsmPath, microvmId); } export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts index 84afb5be3b..2d4252ac71 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -3,18 +3,23 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; import { loadMicrovmProviderConfig } from './config'; -import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; +import { setMicrovmGithubRunnerId } from './runner-metadata'; vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); vi.mock('./microvms', () => ({ isRetryableMicrovmError: vi.fn(), runMicrovmRunner: vi.fn(), - tagMicrovm: vi.fn(), terminateMicrovm: vi.fn(), })); +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + setMicrovmGithubRunnerId: vi.fn(), +})); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const githubClient = {} as Octokit; const createStartRunnerConfig = vi.fn(); @@ -42,9 +47,10 @@ beforeEach(() => { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', maximumDurationInSeconds: 1200, + metadataSsmPath, }); vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); - vi.mocked(tagMicrovm).mockResolvedValue(); + vi.mocked(setMicrovmGithubRunnerId).mockResolvedValue(); vi.mocked(terminateMicrovm).mockResolvedValue(); vi.mocked(isRetryableMicrovmError).mockReturnValue(false); createStartRunnerConfig.mockResolvedValue([]); @@ -83,6 +89,20 @@ describe('createMicrovmRunners', () => { ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); }); + it('rejects a metadata path that overlaps the JIT configuration path', async () => { + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 1200, + metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + }); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + it('classifies invalid provider configuration as non-retryable', async () => { vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { throw new Error('missing image'); @@ -115,9 +135,7 @@ describe('createMicrovmRunners', () => { expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); const options = createStartRunnerConfig.mock.calls[0][3]; expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); - expect(tagMicrovm).toHaveBeenNthCalledWith(1, imageArn, 'mvm-1', { - 'ghr:github_runner_id': 'github-mvm-1', - }); + expect(setMicrovmGithubRunnerId).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', 'github-mvm-1'); }); it('applies dynamic labels to the RunMicrovm configuration and metadata tags', async () => { @@ -143,6 +161,7 @@ describe('createMicrovmRunners', () => { imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', maximumDurationInSeconds: 7200, + metadataSsmPath, }, environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), @@ -150,9 +169,7 @@ describe('createMicrovmRunners', () => { runnerType: 'Org', source: 'scale-up-lambda', }); - expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { - 'ghr:github_runner_id': 'github-mvm-1', - }); + expect(setMicrovmGithubRunnerId).toHaveBeenCalledWith(metadataSsmPath, 'mvm-1', 'github-mvm-1'); }); it('retries a JIT setup failure even when runner cleanup fails', async () => { @@ -163,7 +180,7 @@ describe('createMicrovmRunners', () => { createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); }); it.each([ @@ -186,6 +203,6 @@ describe('createMicrovmRunners', () => { createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); }); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index ca3497dd0e..5393a49d85 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -9,7 +9,8 @@ import type { } from '../../../../core'; import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; import { loadMicrovmProviderConfig } from './config'; -import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerId } from './runner-metadata'; const logger = createChildLogger('microvm-runner-config'); @@ -46,6 +47,7 @@ export async function createMicrovmRunners( let config; try { config = { ...loadMicrovmProviderConfig(), ...overrides }; + assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, githubRunnerConfig.ssmTokenPath); } catch (error) { logger.error('Invalid Lambda MicroVM provider configuration', { error }); return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; @@ -73,14 +75,12 @@ export async function createMicrovmRunners( const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], onJitConfigCreated: async (runnerId, metadata) => { - await tagMicrovm(config.imageIdentifier, runnerId, { - 'ghr:github_runner_id': metadata.githubRunnerId, - }); + await setMicrovmGithubRunnerId(config.metadataSsmPath, runnerId, metadata.githubRunnerId); }, }); if (failedRunnerIds.includes(microvmId)) { - await terminateMicrovm(microvmId).catch((terminationError) => { + await terminateMicrovm(microvmId, config.metadataSsmPath).catch((terminationError) => { logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { error: terminationError, }); @@ -91,7 +91,7 @@ export async function createMicrovmRunners( } } catch (error) { if (microvmId) { - await terminateMicrovm(microvmId).catch((terminationError) => { + await terminateMicrovm(microvmId, config.metadataSsmPath).catch((terminationError) => { logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { error: terminationError, }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts new file mode 100644 index 0000000000..d9db41e4a4 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -0,0 +1,253 @@ +import { deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + assertSeparatedMicrovmMetadataPath, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerMetadata, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + microvmMetadataParameterName, + setMicrovmGithubRunnerId, + setMicrovmOrphan, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + deleteParameter: vi.fn(), + getParametersByPath: vi.fn(), + putParameter: vi.fn(), +})); + +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + createdAt: '2026-08-19T10:00:00.000Z', + expiresAt: '2026-08-19T11:00:00.000Z', + ...overrides, + }; +} + +function states(entries: [string, MicrovmState][]): Map { + return new Map(entries); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useRealTimers(); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(getParametersByPath).mockResolvedValue(new Map()); + vi.mocked(putParameter).mockResolvedValue(); +}); + +describe('MicroVM metadata paths', () => { + it('uses one base parameter per validated MicroVM ID', () => { + expect(microvmMetadataParameterName(`${metadataSsmPath}/`, 'microvm-123')).toBe(`${metadataSsmPath}/microvm-123`); + expect(() => microvmMetadataParameterName(metadataSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + }); + + it('requires metadata to use a prefix separate from JIT configuration', () => { + expect(() => + assertSeparatedMicrovmMetadataPath(metadataSsmPath, '/github-action-runners/unit-test/token'), + ).not.toThrow(); + expect(() => assertSeparatedMicrovmMetadataPath('/runner/token/metadata', '/runner/token')).toThrow( + 'must be separate', + ); + expect(() => assertSeparatedMicrovmMetadataPath('/runner', '/runner/token')).toThrow('must be separate'); + }); +}); + +describe('MicroVM metadata lifecycle', () => { + it('creates non-secret, expiring ownership metadata without overwrite', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T10:00:00.000Z')); + + await createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + maximumDurationInSeconds: 1200, + }); + + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1`, + JSON.stringify(metadata({ expiresAt: '2026-08-19T10:25:00.000Z' })), + false, + ); + }); + + it('loads active metadata with independent state and cleans expired inactive records', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + const active = metadata({ expiresAt: '2026-08-19T12:30:00.000Z' }); + const expiredInactive = metadata({ microvmId: 'mvm-old', expiresAt: '2026-08-19T11:00:00.000Z' }); + const unexpiredInactive = metadata({ microvmId: 'mvm-new', expiresAt: '2026-08-19T12:30:00.000Z' }); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(active)], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.orphan`, 'true'], + [`${metadataSsmPath}/mvm-old`, JSON.stringify(expiredInactive)], + [`${metadataSsmPath}/mvm-new`, JSON.stringify(unexpiredInactive)], + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map([['mvm-1', { ...active, githubRunnerId: 'github-42', orphan: true }]]), + }); + expect(getParametersByPath).toHaveBeenCalledWith(metadataSsmPath); + expect(deleteParameter).toHaveBeenCalledTimes(4); + expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-old`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-new`); + }); + + it('fails closed for invalid metadata or state belonging to an active MicroVM', async () => { + vi.mocked(getParametersByPath).mockResolvedValue(new Map([[`${metadataSsmPath}/mvm-1`, '{not-json']])); + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'invalid ownership metadata', + ); + + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.orphan`, 'invalid'], + ]), + ); + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'invalid orphan state', + ); + }); + + it('propagates metadata path lookup errors so inventory fails closed', async () => { + vi.mocked(getParametersByPath).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'AccessDenied', + ); + }); + + it('updates GitHub and orphan state without a shared read-modify-write record', async () => { + await setMicrovmGithubRunnerId(metadataSsmPath, 'mvm-1', 'github-42'); + expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + + await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); + expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { + overwrite: true, + }); + }); + + it('marks cleanup independently and deletes state before ownership metadata', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + { overwrite: true }, + ); + + await deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1'); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + `${metadataSsmPath}/mvm-1`, + ]); + }); + + it('returns tracked and state-only active cleanup requests for termination retry', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-untracked.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-terminating.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + ]), + ); + + await expect( + listMicrovmRunnerMetadata( + metadataSsmPath, + states([ + ['mvm-1', 'RUNNING'], + ['mvm-untracked', 'PENDING'], + ['mvm-terminating', 'TERMINATING'], + ]), + ), + ).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1', 'mvm-untracked'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('does not starve cleanup requests when more than one reconciliation batch is pending', async () => { + const cleanupIds = Array.from({ length: 11 }, (_, index) => `mvm-cleanup-${index}`); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map( + cleanupIds.map((microvmId) => [ + `${metadataSsmPath}/${microvmId}.cleanup-requested-at`, + '2026-08-19T10:15:00.000Z', + ]), + ), + ); + + await expect( + listMicrovmRunnerMetadata( + metadataSsmPath, + states(cleanupIds.map((microvmId): [string, MicrovmState] => [microvmId, 'RUNNING'])), + ), + ).resolves.toEqual({ cleanupMicrovmIds: cleanupIds, metadataById: new Map() }); + }); + + it('cleans terminal state-only records and aged markers after inventory no longer sees the MicroVM', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-terminal.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-missing.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + [`${metadataSsmPath}/mvm-recent.cleanup-requested-at`, '2026-08-19T11:59:00.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual( + { cleanupMicrovmIds: [], metadataById: new Map() }, + ); + expect(deleteParameter).toHaveBeenCalledTimes(8); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-terminal`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-recent`); + }); + + it('fails closed for active state metadata without ownership or a cleanup request', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42']]), + ); + + await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'state metadata but no ownership metadata', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts new file mode 100644 index 0000000000..3974334e07 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -0,0 +1,348 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; + +import type { LambdaRunnerSource, RunnerType } from '../../../../core'; + +const logger = createChildLogger('microvm-runner-metadata'); + +const METADATA_VERSION = 1; +const EXPIRATION_GRACE_IN_SECONDS = 300; +const MAX_RECONCILED_RUNNERS = 10; +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; +const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; +const ORPHAN_SUFFIX = '.orphan'; +const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerMetadata { + bypassRemoval?: boolean; + createdAt: string; + environment: string; + expiresAt: string; + githubRunnerId?: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + orphan?: boolean; + runnerOwner: string; + runnerType: RunnerType; + source: LambdaRunnerSource; + version: 1; +} + +export interface MicrovmRunnerMetadataInventory { + cleanupMicrovmIds: string[]; + metadataById: Map; +} + +export interface CreateMicrovmRunnerMetadataInput { + environment: string; + imageArn: string; + imageVersion?: string; + maximumDurationInSeconds: number; + microvmId: string; + runnerOwner: string; + runnerType: RunnerType; + source: LambdaRunnerSource; +} + +function normalizedPath(path: string): string { + return path.trim().replace(/\/+$/, ''); +} + +export function microvmMetadataParameterName(metadataSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizedPath(metadataSsmPath)}/${microvmId}`; +} + +function stateParameterName(metadataSsmPath: string, microvmId: string, suffix: string): string { + return `${microvmMetadataParameterName(metadataSsmPath, microvmId)}${suffix}`; +} + +function metadataParameterNames(metadataSsmPath: string, microvmId: string): string[] { + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + return [ + `${baseName}${GITHUB_RUNNER_ID_SUFFIX}`, + `${baseName}${ORPHAN_SUFFIX}`, + `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, + baseName, + ]; +} + +export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerConfigSsmPath: string): void { + const metadataPath = normalizedPath(metadataSsmPath); + const runnerConfigPath = normalizedPath(runnerConfigSsmPath); + if ( + metadataPath === runnerConfigPath || + metadataPath.startsWith(`${runnerConfigPath}/`) || + runnerConfigPath.startsWith(`${metadataPath}/`) + ) { + throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT configuration path'); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length > 0); +} + +function optionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === 'boolean'; +} + +function parseMetadata(value: string, expectedMicrovmId: string): MicrovmRunnerMetadata | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + + if (!isRecord(parsed)) return undefined; + + const createdAt = typeof parsed.createdAt === 'string' ? Date.parse(parsed.createdAt) : Number.NaN; + const expiresAt = typeof parsed.expiresAt === 'string' ? Date.parse(parsed.expiresAt) : Number.NaN; + if ( + parsed.version !== METADATA_VERSION || + parsed.microvmId !== expectedMicrovmId || + typeof parsed.environment !== 'string' || + parsed.environment.length === 0 || + typeof parsed.runnerOwner !== 'string' || + parsed.runnerOwner.length === 0 || + (parsed.runnerType !== 'Org' && parsed.runnerType !== 'Repo') || + (parsed.source !== 'scale-up-lambda' && parsed.source !== 'pool-lambda') || + typeof parsed.imageArn !== 'string' || + parsed.imageArn.length === 0 || + !optionalString(parsed.imageVersion) || + !optionalBoolean(parsed.bypassRemoval) || + !Number.isFinite(createdAt) || + !Number.isFinite(expiresAt) || + expiresAt <= createdAt + ) { + return undefined; + } + + return { + version: METADATA_VERSION, + microvmId: expectedMicrovmId, + environment: parsed.environment, + runnerOwner: parsed.runnerOwner, + runnerType: parsed.runnerType, + source: parsed.source, + imageArn: parsed.imageArn, + imageVersion: parsed.imageVersion, + bypassRemoval: parsed.bypassRemoval, + createdAt: parsed.createdAt as string, + expiresAt: parsed.expiresAt as string, + }; +} + +export async function createMicrovmRunnerMetadata( + metadataSsmPath: string, + input: CreateMicrovmRunnerMetadataInput, +): Promise { + const createdAt = new Date(); + const metadata: MicrovmRunnerMetadata = { + version: METADATA_VERSION, + microvmId: input.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.imageArn, + imageVersion: input.imageVersion, + createdAt: createdAt.toISOString(), + expiresAt: new Date( + createdAt.getTime() + (input.maximumDurationInSeconds + EXPIRATION_GRACE_IN_SECONDS) * 1000, + ).toISOString(), + }; + + await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false); +} + +function invalidStateReason(parameters: Map, baseName: string): string | undefined { + const orphan = parameters.get(`${baseName}${ORPHAN_SUFFIX}`); + if (orphan !== undefined && orphan !== 'true' && orphan !== 'false') return 'invalid orphan state'; + + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (cleanupRequestedAt !== undefined && !Number.isFinite(Date.parse(cleanupRequestedAt))) { + return 'invalid cleanup request timestamp'; + } + return undefined; +} + +function shouldDeleteMetadata( + metadata: MicrovmRunnerMetadata, + state: MicrovmState | undefined, + cleanupRequestedAt: string | undefined, + now: number, +): boolean { + if (state === 'TERMINATED') return true; + if (state !== undefined) return false; + + const cleanupGraceElapsed = + cleanupRequestedAt !== undefined && Date.parse(cleanupRequestedAt) + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now; + return cleanupGraceElapsed || Date.parse(metadata.expiresAt) <= now; +} + +export async function listMicrovmRunnerMetadata( + metadataSsmPath: string, + microvmStates: ReadonlyMap, +): Promise { + const metadataById = new Map(); + const cleanupMicrovmIds = new Set(); + const parameters = await getParametersByPath(normalizedPath(metadataSsmPath)); + const parameterPrefix = `${normalizedPath(metadataSsmPath)}/`; + const now = Date.now(); + const metadataBaseIds = new Set(); + const stateParameterIds = new Set(); + const runnersToDelete = new Set(); + + for (const parameterName of parameters.keys()) { + if (!parameterName.startsWith(parameterPrefix)) continue; + for (const suffix of [GITHUB_RUNNER_ID_SUFFIX, ORPHAN_SUFFIX, CLEANUP_REQUESTED_AT_SUFFIX]) { + if (!parameterName.endsWith(suffix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length, -suffix.length); + if (MICROVM_ID_PATTERN.test(microvmId)) stateParameterIds.add(microvmId); + break; + } + } + + for (const [parameterName, value] of parameters) { + if (!parameterName.startsWith(parameterPrefix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length); + if (!MICROVM_ID_PATTERN.test(microvmId)) continue; + metadataBaseIds.add(microvmId); + + const state = microvmStates.get(microvmId); + const metadata = parseMetadata(value, microvmId); + if (!metadata) { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); + } + if (state === 'TERMINATED') runnersToDelete.add(microvmId); + else logger.warn(`Ignoring invalid MicroVM runner metadata for '${microvmId}'`); + continue; + } + + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const stateError = invalidStateReason(parameters, baseName); + if (stateError) { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has ${stateError}`); + } + if (state === 'TERMINATED' || (state === undefined && Date.parse(metadata.expiresAt) <= now)) { + runnersToDelete.add(microvmId); + } + logger.warn(`Ignoring MicroVM runner metadata for '${microvmId}' with ${stateError}`); + continue; + } + + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (shouldDeleteMetadata(metadata, state, cleanupRequestedAt, now)) { + runnersToDelete.add(microvmId); + continue; + } + + if (state === undefined || !ACTIVE_STATES.has(state)) continue; + + if (cleanupRequestedAt !== undefined) { + cleanupMicrovmIds.add(microvmId); + continue; + } + + metadataById.set(microvmId, { + ...metadata, + githubRunnerId: parameters.get(`${baseName}${GITHUB_RUNNER_ID_SUFFIX}`), + orphan: parameters.get(`${baseName}${ORPHAN_SUFFIX}`) === 'true', + }); + } + + for (const microvmId of stateParameterIds) { + if (metadataBaseIds.has(microvmId)) continue; + + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const state = microvmStates.get(microvmId); + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + const stateError = invalidStateReason(parameters, baseName); + + if (stateError && state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has ${stateError}`); + } + if (state !== undefined && ACTIVE_STATES.has(state)) { + if (cleanupRequestedAt === undefined) { + throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); + } + cleanupMicrovmIds.add(microvmId); + continue; + } + if (state === 'TERMINATED') { + runnersToDelete.add(microvmId); + continue; + } + if (state === undefined) { + const cleanupGraceElapsed = + cleanupRequestedAt !== undefined && + Number.isFinite(Date.parse(cleanupRequestedAt)) && + Date.parse(cleanupRequestedAt) + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now; + if (cleanupRequestedAt === undefined || stateError !== undefined || cleanupGraceElapsed) { + runnersToDelete.add(microvmId); + } + } + } + + for (const microvmId of [...runnersToDelete].slice(0, MAX_RECONCILED_RUNNERS)) { + try { + await deleteMicrovmRunnerMetadata(metadataSsmPath, microvmId); + } catch (error) { + logger.warn(`Failed to delete reconciled MicroVM runner metadata '${microvmId}'`, { error }); + } + } + + return { + cleanupMicrovmIds: [...cleanupMicrovmIds], + metadataById, + }; +} + +export async function setMicrovmGithubRunnerId( + metadataSsmPath: string, + microvmId: string, + githubRunnerId: string, +): Promise { + if (!githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + await putParameter(stateParameterName(metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), githubRunnerId, false, { + overwrite: true, + }); +} + +export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: string, orphan: boolean): Promise { + await putParameter(stateParameterName(metadataSsmPath, microvmId, ORPHAN_SUFFIX), String(orphan), false, { + overwrite: true, + }); +} + +export async function markMicrovmCleanupPending(metadataSsmPath: string, microvmId: string): Promise { + await putParameter( + stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), + new Date().toISOString(), + false, + { overwrite: true }, + ); +} + +export async function deleteMicrovmRunnerMetadata(metadataSsmPath: string, microvmId: string): Promise { + for (const parameterName of metadataParameterNames(metadataSsmPath, microvmId)) { + try { + await deleteParameter(parameterName); + } catch (error) { + if (!(error instanceof Error && error.name === 'ParameterNotFound')) throw error; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index 613364e7d2..fce2d06137 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -1,24 +1,25 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { loadMicrovmProviderConfig } from './config'; -import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; import { createMicrovmScaleDownProvider } from './scale-down'; +import { setMicrovmOrphan } from './runner-metadata'; vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn(), microvmBootTimeExceeded: vi.fn(), - tagMicrovm: vi.fn(), terminateMicrovm: vi.fn(), - untagMicrovm: vi.fn(), })); +vi.mock('./runner-metadata', () => ({ setMicrovmOrphan: vi.fn() })); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; -const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const providerConfig = { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', maximumDurationInSeconds: 1200, + metadataSsmPath, }; beforeEach(() => { @@ -26,8 +27,7 @@ beforeEach(() => { vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); vi.mocked(listMicrovmRunners).mockResolvedValue([]); vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); - vi.mocked(tagMicrovm).mockResolvedValue(); - vi.mocked(untagMicrovm).mockResolvedValue(); + vi.mocked(setMicrovmOrphan).mockResolvedValue(); vi.mocked(terminateMicrovm).mockResolvedValue(); }); @@ -38,30 +38,34 @@ describe('createMicrovmScaleDownProvider', () => { await provider.list('unit-test'); await provider.list('unit-test', true); - expect(listMicrovmRunners).toHaveBeenNthCalledWith(1, { - environment: 'unit-test', - orphan: undefined, - }); - expect(listMicrovmRunners).toHaveBeenNthCalledWith(2, { - environment: 'unit-test', - orphan: true, - }); + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 1, + { + environment: 'unit-test', + orphan: undefined, + }, + metadataSsmPath, + ); + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 2, + { + environment: 'unit-test', + orphan: true, + }, + metadataSsmPath, + ); }); - it('uses the listed image ARN when marking, unmarking, and terminating runners', async () => { - vi.mocked(listMicrovmRunners).mockResolvedValue([ - { id: 'mvm-1', imageArn: overrideImageArn, owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, - ]); + it('uses durable metadata when marking, unmarking, and terminating runners', async () => { const provider = createMicrovmScaleDownProvider(); - await provider.list('unit-test'); await provider.markOrphan('mvm-1'); await provider.unmarkOrphan('mvm-1'); await provider.terminate('mvm-1'); - expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { 'ghr:orphan': 'true' }); - expect(untagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', ['ghr:orphan']); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', true); + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(2, metadataSsmPath, 'mvm-1', false); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); }); it('uses the MicroVM boot-time policy', () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts index 9ea9dc474a..9cda68cf53 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -1,28 +1,21 @@ import type { ScaleDownComputeProvider } from '../../../../core'; import { loadMicrovmProviderConfig } from './config'; import type { MicrovmRunnerInfo } from './microvms'; -import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { setMicrovmOrphan } from './runner-metadata'; export function createMicrovmScaleDownProvider(): Omit { - const imageArnByRunnerId = new Map(); + const metadataSsmPath = () => loadMicrovmProviderConfig().metadataSsmPath; async function list(environment: string, orphan?: boolean): Promise { - const runners = await listMicrovmRunners({ environment, orphan }); - for (const runner of runners) { - if (runner.imageArn) imageArnByRunnerId.set(runner.id, runner.imageArn); - } - return runners; - } - - function imageArnForRunner(id: string): string { - return imageArnByRunnerId.get(id) ?? loadMicrovmProviderConfig().imageIdentifier; + return await listMicrovmRunners({ environment, orphan }, metadataSsmPath()); } return { list, bootTimeExceeded: microvmBootTimeExceeded, - markOrphan: async (id) => await tagMicrovm(imageArnForRunner(id), id, { 'ghr:orphan': 'true' }), - unmarkOrphan: async (id) => await untagMicrovm(imageArnForRunner(id), id, ['ghr:orphan']), - terminate: terminateMicrovm, + markOrphan: async (id) => await setMicrovmOrphan(metadataSsmPath(), id, true), + unmarkOrphan: async (id) => await setMicrovmOrphan(metadataSsmPath(), id, false), + terminate: async (id) => await terminateMicrovm(id, metadataSsmPath()), }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 91c1931f83..06668b935f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -10,6 +10,7 @@ declare global { MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; MICROVM_LOG_GROUP: string | undefined; MICROVM_MAXIMUM_DURATION_IN_SECONDS: string | undefined; + MICROVM_METADATA_SSM_PATH: string; } } } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts deleted file mode 100644 index 9785383727..0000000000 --- a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels-policy.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { AwsDynamicLabelsPolicy } from '../../../../contracts'; - -function globToRegExp(glob: string): RegExp { - const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); - return new RegExp(`^${pattern}$`); -} - -function matchesAny(value: string, patterns: string[] | undefined): boolean { - if (!patterns || patterns.length === 0) return false; - return patterns.some((pattern) => globToRegExp(pattern).test(value)); -} - -function evaluateLabel(label: string, policy: AwsDynamicLabelsPolicy, labelPrefix: string): string | null { - const stripped = label.slice(labelPrefix.length); - const colonIndex = stripped.indexOf(':'); - const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); - const value = colonIndex === -1 ? undefined : stripped.slice(colonIndex + 1); - - if (policy.blocked_keys?.includes(key)) { - return `key '${key}' is in blocked_keys`; - } - - const rule = policy.restricted_keys?.[key]; - if (!rule || value === undefined) return null; - - if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { - return `value '${value}' not in allowed list`; - } - if (rule.denied && matchesAny(value, rule.denied)) { - return `value '${value}' in denied list`; - } - if (rule.max !== undefined && rule.max !== null) { - const valueNumber = Number(value); - const maximum = Number(rule.max); - if (!Number.isFinite(valueNumber) || !Number.isFinite(maximum)) { - return `max set but value '${value}' or max '${rule.max}' is not numeric`; - } - if (valueNumber > maximum) { - return `value '${value}' exceeds max '${rule.max}'`; - } - } - - return null; -} - -export function violationsAgainstAwsDynamicLabelsPolicy( - labels: string[], - policy: AwsDynamicLabelsPolicy | null | undefined, - labelPrefix: string, -): { label: string; reason: string }[] { - if (!policy) return []; - - const violations: { label: string; reason: string }[] = []; - for (const label of labels) { - if (!label.startsWith(labelPrefix)) continue; - const reason = evaluateLabel(label, policy, labelPrefix); - if (reason) violations.push({ label, reason }); - } - return violations; -} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts index 6b157309af..b21ad792f3 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -9,6 +9,13 @@ const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connec describe('microvmDynamicLabelProvider', () => { it('accepts supported MicroVM overrides', () => { const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { allowed: [egressConnectorArn] }, + 'image-arn': { allowed: [imageArn] }, + 'image-version': { allowed: ['3.0'] }, + }, + }; const dynamicLabels = [ `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, `ghr-microvm-image-arn:${imageArn}`, @@ -16,21 +23,40 @@ describe('microvmDynamicLabelProvider', () => { 'ghr-microvm-maximum-duration-in-seconds:7200', ]; - expect(selectQueue(queue, dynamicLabels)).toEqual({ - queue, - labels: ['self-hosted', 'linux', ...dynamicLabels], - }); + expect(getViolations(queue, dynamicLabels)).toEqual([]); }); - it('rejects dynamic labels when the queue disables them', () => { - const queue = microvmQueue(); - queue.matcherConfig.enableDynamicLabels = false; - - expect(selectQueue(queue, ['ghr-microvm-image-version:3.0'])).toBeUndefined(); + it('requires explicit allowlists for image code and network-boundary overrides', () => { + expect( + getViolations(microvmQueue(), [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:3600', + ]), + ).toEqual([ + { + label: `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + reason: "key 'egress-network-connectors' requires an explicit allowed list", + }, + { + label: `ghr-microvm-image-arn:${imageArn}`, + reason: "key 'image-arn' requires an explicit allowed list", + }, + { + label: 'ghr-microvm-image-version:3.0', + reason: "key 'image-version' requires an explicit allowed list", + }, + ]); }); - it('rejects unsupported MicroVM resource overrides', () => { - expect(selectQueue(microvmQueue(), ['ghr-microvm-memory:8192'])).toBeUndefined(); + it('preserves violations from the MicroVM label parser', () => { + expect(getViolations(microvmQueue(), ['ghr-microvm-memory:8192'])).toEqual([ + { + label: 'ghr-microvm-memory:8192', + reason: "key 'memory' is not a supported MicroVM override", + }, + ]); }); it('enforces the AWS dynamic-label policy', () => { @@ -39,7 +65,12 @@ describe('microvmDynamicLabelProvider', () => { restricted_keys: { 'maximum-duration-in-seconds': { max: 3600 } }, }; - expect(selectQueue(queue, ['ghr-microvm-maximum-duration-in-seconds:7200'])).toBeUndefined(); + expect(getViolations(queue, ['ghr-microvm-maximum-duration-in-seconds:7200'])).toEqual([ + { + label: 'ghr-microvm-maximum-duration-in-seconds:7200', + reason: "value '7200' exceeds max '3600'", + }, + ]); }); it('applies allowed patterns to the complete image ARN', () => { @@ -53,11 +84,13 @@ describe('microvmDynamicLabelProvider', () => { }; expect( - selectQueue(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large']), - ).toBeDefined(); + getViolations(queue, [ + 'ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large', + ]), + ).toEqual([]); expect( - selectQueue(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), - ).toBeUndefined(); + getViolations(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toHaveLength(1); }); it('applies the policy to each egress connector label', () => { @@ -71,23 +104,22 @@ describe('microvmDynamicLabelProvider', () => { }; expect( - selectQueue(queue, [ + getViolations(queue, [ 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', ]), - ).toBeDefined(); + ).toEqual([]); expect( - selectQueue(queue, [ + getViolations(queue, [ 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', ]), - ).toBeUndefined(); + ).toHaveLength(1); }); }); -function selectQueue(queue: RunnerMatcherConfig, sanitizedGhrLabels: string[]) { - return microvmDynamicLabelProvider.selectQueue({ +function getViolations(queue: RunnerMatcherConfig, labels: string[]) { + return microvmDynamicLabelProvider.getViolations({ queue, - nonGhrLabels: ['self-hosted', 'linux'], - sanitizedGhrLabels, + labels, }); } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts index 36eb3e7670..e7c5485617 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -1,48 +1,32 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; - -import type { DynamicLabelDispatchTarget, DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; +import type { DynamicLabelProvider } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; -import { violationsAgainstAwsDynamicLabelsPolicy } from './dynamic-labels-policy'; - -const logger = createChildLogger('handler'); -export function selectMicrovmDynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - if (!queue.matcherConfig.enableDynamicLabels) { - logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); - continue; - } +const RESOURCE_BOUNDARY_KEYS = new Set(['egress-network-connectors', 'image-arn', 'image-version']); - const parsedLabels = parseMicrovmDynamicLabels(sanitizedGhrLabels); - const policyViolations = violationsAgainstAwsDynamicLabelsPolicy( - sanitizedGhrLabels, - queue.matcherConfig.awsDynamicLabelsPolicy, - MICROVM_DYNAMIC_LABEL_PREFIX, - ); - const violations = [...parsedLabels.violations, ...policyViolations]; +function resourceBoundaryViolations( + labels: string[], + policy: Parameters[1], +) { + return labels.flatMap((label) => { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) return []; - if (violations.length === 0) { - return { - queue, - labels: [...nonGhrLabels, ...sanitizedGhrLabels], - }; - } + const key = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length).split(':', 1)[0]; + if (!RESOURCE_BOUNDARY_KEYS.has(key) || policy?.blocked_keys?.includes(key)) return []; - for (const violation of violations) { - logger.warn( - `Queue ${queue.id}: dynamic label '${violation.label}' is not accepted (${violation.reason}); trying next match`, - ); - } - } - - return undefined; + const allowed = policy?.restricted_keys?.[key]?.allowed; + return allowed && allowed.length > 0 ? [] : [{ label, reason: `key '${key}' requires an explicit allowed list` }]; + }); } export const microvmDynamicLabelProvider: DynamicLabelProvider = { - selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => - selectMicrovmDynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), + getViolations: ({ queue, labels }) => [ + ...parseMicrovmDynamicLabels(labels).violations, + ...resourceBoundaryViolations(labels, queue.matcherConfig.awsDynamicLabelsPolicy), + ...violationsAgainstAwsDynamicLabelsPolicy( + labels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ), + ], }; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts index bdc6d2918c..efe6bcd070 100644 --- a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -1,36 +1,27 @@ -import { describe, expect, it } from 'vitest'; - -import type { RunnerMatcherConfig } from '../../contracts'; +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; import { provider } from './webhook'; -describe('MicroVM webhook provider contract', () => { - it('exposes MicroVM dynamic-label selection', () => { - const plugin = provider.createPlugin(); - const queue = microvmQueue(); - - expect(plugin.type).toBe('microvm'); - expect( - plugin.capabilities.dynamicLabels.selectQueue({ - queue, - nonGhrLabels: ['self-hosted', 'linux'], - sanitizedGhrLabels: ['ghr-microvm-image-version:3.0'], - }), - ).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-microvm-image-version:3.0'], - }); - }); -}); - -function microvmQueue(): RunnerMatcherConfig { - return { - id: 'microvm', - arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', - computeProvider: 'microvm', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-microvm-maximum-duration-in-seconds:3600'], + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['maximum-duration-in-seconds'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'maximum-duration-in-seconds': { max: 1800 }, + }, + }; + }, }, - }; -} + ], +}); From a000a3f58e157166ce55360df7aa96d370b36eb3 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 23:09:01 +0200 Subject: [PATCH 15/21] fix(compute-providers): make metadata cleanup idempotent --- .../src/control-plane/runner-metadata.test.ts | 32 +++++++++++++++++++ .../src/control-plane/runner-metadata.ts | 9 +++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts index d9db41e4a4..10a1fa9349 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -175,6 +175,38 @@ describe('MicroVM metadata lifecycle', () => { ]); }); + it('continues deleting metadata when optional parameters are already absent', async () => { + vi.mocked(deleteParameter) + .mockRejectedValueOnce( + Object.assign(new Error('ParameterNotFound'), { + __type: 'ParameterNotFound', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }), + ) + .mockRejectedValueOnce(Object.assign(new Error('missing parameter'), { name: 'ParameterNotFound' })); + + await expect(deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + `${metadataSsmPath}/mvm-1`, + ]); + }); + + it('propagates metadata deletion failures other than missing parameters', async () => { + const error = Object.assign(new Error('AccessDeniedException'), { + __type: 'AccessDeniedException', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }); + vi.mocked(deleteParameter).mockRejectedValueOnce(error); + + await expect(deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1')).rejects.toBe(error); + expect(deleteParameter).toHaveBeenCalledTimes(1); + }); + it('returns tracked and state-only active cleanup requests for termination retry', async () => { vi.mocked(getParametersByPath).mockResolvedValue( new Map([ diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index 3974334e07..965b0222d8 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -88,6 +88,13 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function isParameterNotFound(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === 'ParameterNotFound' || ('__type' in error && error.__type === 'ParameterNotFound')) + ); +} + function optionalString(value: unknown): value is string | undefined { return value === undefined || (typeof value === 'string' && value.length > 0); } @@ -342,7 +349,7 @@ export async function deleteMicrovmRunnerMetadata(metadataSsmPath: string, micro try { await deleteParameter(parameterName); } catch (error) { - if (!(error instanceof Error && error.name === 'ParameterNotFound')) throw error; + if (!isParameterNotFound(error)) throw error; } } } From 9bf444e0ac847bb41a6d64fdbbb4904f85bf6fce Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 23:41:22 +0200 Subject: [PATCH 16/21] fix(compute-providers): remove MicroVM duration label --- .../compute-providers/aws/microvm/README.md | 16 +++++-------- .../src/control-plane/runner-config.test.ts | 5 ++-- .../src/control-plane/scale-up.test.ts | 15 ++++++------ .../aws/microvm/src/dynamic-labels.test.ts | 8 +++---- .../aws/microvm/src/dynamic-labels.ts | 14 ----------- .../src/webhook/dynamic-labels.test.ts | 24 +++++++++++-------- .../aws/microvm/webhook.test.ts | 13 +++++++--- .../test/webhook-provider-contract.ts | 18 ++++++++++---- 8 files changed, 57 insertions(+), 56 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index 71e53f9f7b..87a1af6e50 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -60,12 +60,11 @@ separate roles, prefixes, and provider deployments. When a runner matcher enables dynamic labels, workflow jobs can override the following `RunMicrovm` inputs: -| Label | Override | -| --------------------------------------------------- | ---------------------------------------------- | -| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | -| `ghr-microvm-image-arn:` | MicroVM image ARN | -| `ghr-microvm-image-version:` | MicroVM image version | -| `ghr-microvm-maximum-duration-in-seconds:` | Maximum lifetime from 1 through 28,800 seconds | +| Label | Override | +| --------------------------------------------- | -------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | Repeat `ghr-microvm-egress-network-connectors:` to attach multiple connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These @@ -84,7 +83,7 @@ explicit `allowed` list for the corresponding key. Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from workflow jobs. The MicroVM policy keys are `egress-network-connectors`, -`image-arn`, `image-version`, and `maximum-duration-in-seconds`. For example: +`image-arn`, and `image-version`. For example: ```json { @@ -97,9 +96,6 @@ workflow jobs. The MicroVM policy keys are `egress-network-connectors`, }, "image-version": { "allowed": ["3.*"] - }, - "maximum-duration-in-seconds": { - "max": 3600 } } } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts index 2d4252ac71..940b1e2771 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -138,7 +138,7 @@ describe('createMicrovmRunners', () => { expect(setMicrovmGithubRunnerId).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', 'github-mvm-1'); }); - it('applies dynamic labels to the RunMicrovm configuration and metadata tags', async () => { + it('applies dynamic labels without overriding the deployment-controlled duration', async () => { const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; const overrideEgressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; @@ -151,7 +151,6 @@ describe('createMicrovmRunners', () => { egressNetworkConnectors: [overrideEgressConnectorArn], imageIdentifier: overrideImageArn, imageVersion: '3.0', - maximumDurationInSeconds: 7200, }); expect(runMicrovmRunner).toHaveBeenCalledWith({ @@ -160,7 +159,7 @@ describe('createMicrovmRunners', () => { imageIdentifier: overrideImageArn, imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - maximumDurationInSeconds: 7200, + maximumDurationInSeconds: 1200, metadataSsmPath, }, environment: 'unit-test', diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts index cfbbda3257..bd10efd41e 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -50,32 +50,33 @@ describe('createMicrovmScaleUpProvider', () => { `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, `ghr-microvm-image-arn:${overrideImageArn}`, 'ghr-microvm-image-version:3.0', - 'ghr-microvm-maximum-duration-in-seconds:7200', ]), ).resolves.toEqual({ runnerLabels: [ `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, `ghr-microvm-image-arn:${overrideImageArn}`, 'ghr-microvm-image-version:3.0', - 'ghr-microvm-maximum-duration-in-seconds:7200', ], state: { overrides: { egressNetworkConnectors: [overrideEgressConnectorArn], imageIdentifier: overrideImageArn, imageVersion: '3.0', - maximumDurationInSeconds: 7200, }, }, }); }); - it('rejects unsupported MicroVM override labels at the control-plane boundary', async () => { + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects unsupported MicroVM override label %s at the control-plane boundary', async (label, reason) => { const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); - await expect(provider.resolveLabelsForRunners(['ghr-microvm-memory:8192'])).rejects.toThrow( - "key 'memory' is not a supported MicroVM override", - ); + await expect(provider.resolveLabelsForRunners([label])).rejects.toThrow(reason); }); it('counts managed MicroVMs for the runner owner', async () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts index 6ea669a143..442986a0f8 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -15,14 +15,12 @@ describe('parseMicrovmDynamicLabels', () => { `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, `ghr-microvm-image-arn:${imageArn}`, 'ghr-microvm-image-version:3.0', - 'ghr-microvm-maximum-duration-in-seconds:7200', ]), ).toEqual({ overrides: { egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], imageIdentifier: imageArn, imageVersion: '3.0', - maximumDurationInSeconds: 7200, }, violations: [], }); @@ -40,8 +38,10 @@ describe('parseMicrovmDynamicLabels', () => { ], ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], ['ghr-microvm-image-version:', "key 'image-version' requires a value"], - ['ghr-microvm-maximum-duration-in-seconds:0', 'maximum duration must be an integer between 1 and 28800'], - ['ghr-microvm-maximum-duration-in-seconds:28801', 'maximum duration must be an integer between 1 and 28800'], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], ])('rejects invalid override %s', (label, reason) => { const result = parseMicrovmDynamicLabels([label]); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts index 50c223cfd2..2851716149 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -1,6 +1,5 @@ export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; -const MAXIMUM_DURATION_IN_SECONDS = 28_800; const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = @@ -10,7 +9,6 @@ export interface MicrovmDynamicLabelOverrides { egressNetworkConnectors?: string[]; imageIdentifier?: string; imageVersion?: string; - maximumDurationInSeconds?: number; } export interface MicrovmDynamicLabelViolation { @@ -69,18 +67,6 @@ export function parseMicrovmDynamicLabels(labels: string[]): { case 'image-version': overrides.imageVersion = value; break; - case 'maximum-duration-in-seconds': { - const duration = Number(value); - if (!Number.isInteger(duration) || duration < 1 || duration > MAXIMUM_DURATION_IN_SECONDS) { - violations.push({ - label, - reason: `maximum duration must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, - }); - } else { - overrides.maximumDurationInSeconds = duration; - } - break; - } default: violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts index b21ad792f3..2b7b85d74e 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -20,7 +20,6 @@ describe('microvmDynamicLabelProvider', () => { `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, `ghr-microvm-image-arn:${imageArn}`, 'ghr-microvm-image-version:3.0', - 'ghr-microvm-maximum-duration-in-seconds:7200', ]; expect(getViolations(queue, dynamicLabels)).toEqual([]); @@ -32,7 +31,6 @@ describe('microvmDynamicLabelProvider', () => { `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, `ghr-microvm-image-arn:${imageArn}`, 'ghr-microvm-image-version:3.0', - 'ghr-microvm-maximum-duration-in-seconds:3600', ]), ).toEqual([ { @@ -50,11 +48,17 @@ describe('microvmDynamicLabelProvider', () => { ]); }); - it('preserves violations from the MicroVM label parser', () => { - expect(getViolations(microvmQueue(), ['ghr-microvm-memory:8192'])).toEqual([ + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('preserves the parser violation for %s', (label, reason) => { + expect(getViolations(microvmQueue(), [label])).toEqual([ { - label: 'ghr-microvm-memory:8192', - reason: "key 'memory' is not a supported MicroVM override", + label, + reason, }, ]); }); @@ -62,13 +66,13 @@ describe('microvmDynamicLabelProvider', () => { it('enforces the AWS dynamic-label policy', () => { const queue = microvmQueue(); queue.matcherConfig.awsDynamicLabelsPolicy = { - restricted_keys: { 'maximum-duration-in-seconds': { max: 3600 } }, + restricted_keys: { 'image-version': { allowed: ['2.*'] } }, }; - expect(getViolations(queue, ['ghr-microvm-maximum-duration-in-seconds:7200'])).toEqual([ + expect(getViolations(queue, ['ghr-microvm-image-version:3.0'])).toEqual([ { - label: 'ghr-microvm-maximum-duration-in-seconds:7200', - reason: "value '7200' exceeds max '3600'", + label: 'ghr-microvm-image-version:3.0', + reason: "value '3.0' not in allowed list", }, ]); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts index efe6bcd070..ad3baed78b 100644 --- a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -3,13 +3,20 @@ import { provider } from './webhook'; defineWebhookProviderContractTests({ provider, - acceptedDynamicLabels: ['ghr-microvm-maximum-duration-in-seconds:3600'], + acceptedDynamicLabels: ['ghr-microvm-image-version:3.0'], + configureQueue: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['3.0'] }, + }, + }; + }, rejectingPolicies: [ { name: 'blocked keys', apply: (queue) => { queue.matcherConfig.awsDynamicLabelsPolicy = { - blocked_keys: ['maximum-duration-in-seconds'], + blocked_keys: ['image-version'], }; }, }, @@ -18,7 +25,7 @@ defineWebhookProviderContractTests({ apply: (queue) => { queue.matcherConfig.awsDynamicLabelsPolicy = { restricted_keys: { - 'maximum-duration-in-seconds': { max: 1800 }, + 'image-version': { allowed: ['2.*'] }, }, }; }, diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index dd4e3097b0..a6d01b7e6e 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -13,17 +13,25 @@ interface RejectingPolicyCase { interface WebhookProviderContractOptions { provider: WebhookProviderModule; acceptedDynamicLabels: readonly [string, ...string[]]; + configureQueue?(queue: RunnerMatcherConfig): void; rejectingPolicies: readonly [RejectingPolicyCase, ...RejectingPolicyCase[]]; } export function defineWebhookProviderContractTests({ provider, acceptedDynamicLabels, + configureQueue, rejectingPolicies, }: WebhookProviderContractOptions): void { const nonGhrLabels = ['self-hosted', 'linux']; const dynamicLabels = [...acceptedDynamicLabels]; + function configuredRunnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + const queue = runnerQueue(id, computeProvider); + configureQueue?.(queue); + return queue; + } + function expectProviderSelected(queue: RunnerMatcherConfig) { expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toEqual({ queue, @@ -33,11 +41,11 @@ export function defineWebhookProviderContractTests { it('selects an explicitly configured provider through the production registry', () => { - expectProviderSelected(runnerQueue(`${provider.type}-configured`, provider.type)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-configured`, provider.type)); }); it('skips the provider when dynamic labels are disabled', () => { - const queue = runnerQueue(`${provider.type}-disabled`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-disabled`, provider.type); queue.matcherConfig.enableDynamicLabels = false; expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -45,7 +53,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-policy-rejected`, provider.type); policy.apply(queue); expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -53,7 +61,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-normalized`); + const queue = configuredRunnerQueue(`${provider.type}-normalized`); (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; expectProviderSelected(queue); @@ -61,7 +69,7 @@ export function defineWebhookProviderContractTests { - expectProviderSelected(runnerQueue(`${provider.type}-default`)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-default`)); }); } }); From fbf3556316a292debb92340ab375e8cdc0c6fba3 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 20 Aug 2026 16:27:14 +0200 Subject: [PATCH 17/21] fix(compute-providers): fix MicroVM lifetime at eight hours --- .../compute-providers/aws/microvm/README.md | 3 ++- .../microvm/src/control-plane/config.test.ts | 14 +----------- .../aws/microvm/src/control-plane/config.ts | 22 ------------------- .../aws/microvm/src/control-plane/lifetime.ts | 1 + .../src/control-plane/microvms.test.ts | 8 +++---- .../aws/microvm/src/control-plane/microvms.ts | 4 ++-- .../src/control-plane/runner-config.test.ts | 5 +---- .../src/control-plane/runner-metadata.test.ts | 3 +-- .../src/control-plane/runner-metadata.ts | 4 ++-- .../src/control-plane/scale-down.test.ts | 1 - .../aws/microvm/src/environment.d.ts | 1 - 11 files changed, 14 insertions(+), 52 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index 87a1af6e50..b2486b695f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -29,10 +29,11 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_IMAGE_VERSION` (optional) - `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) -- `MICROVM_MAXIMUM_DURATION_IN_SECONDS` (optional, defaults to 3600) - `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) - `MICROVM_LOG_GROUP` (optional) +Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). + The control-plane role requires `ssm:GetParametersByPath`, `ssm:PutParameter`, and `ssm:DeleteParameter` on the dedicated metadata prefix, plus `lambda:ListMicrovms`, `lambda:RunMicrovm`, and `lambda:TerminateMicrovm` for diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index 1cc240a9f7..b68fdffb7c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -12,7 +12,6 @@ beforeEach(() => { delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; - delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; delete process.env.MICROVM_LOG_GROUP; }); @@ -24,24 +23,21 @@ describe('loadMicrovmProviderConfig', () => { executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, - maximumDurationInSeconds: 3600, metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', logging: undefined, }); }); - it('loads versions, logging, duration, and either connector list format', () => { + it('loads versions, logging, and either connector list format', () => { process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; - process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; expect(loadMicrovmProviderConfig()).toMatchObject({ imageVersion: '3.0', ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], - maximumDurationInSeconds: 1200, logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }); }); @@ -58,14 +54,6 @@ describe('loadMicrovmProviderConfig', () => { ); }); - it.each(['0', '28801', '1.5', 'invalid'])('rejects invalid maximum duration %s', (duration) => { - process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = duration; - - expect(() => loadMicrovmProviderConfig()).toThrow( - 'MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and 28800', - ); - }); - it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index ddd7891362..8eacada06c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -1,8 +1,5 @@ import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; -const DEFAULT_MAXIMUM_DURATION_IN_SECONDS = 3600; -const MAXIMUM_DURATION_IN_SECONDS = 28800; - export interface MicrovmProviderConfig { egressNetworkConnectors?: string[]; executionRoleArn: string; @@ -10,7 +7,6 @@ export interface MicrovmProviderConfig { imageVersion?: string; ingressNetworkConnectors?: string[]; logging?: Logging; - maximumDurationInSeconds: number; metadataSsmPath: string; } @@ -59,23 +55,6 @@ function parseNetworkConnectors(name: string, value: string | undefined): string return connectors.map((connector) => connector.trim()); } -function parseMaximumDuration(value: string | undefined): number { - if (!optionalEnvironmentValue(value)) return DEFAULT_MAXIMUM_DURATION_IN_SECONDS; - - const maximumDurationInSeconds = Number(value); - if ( - !Number.isInteger(maximumDurationInSeconds) || - maximumDurationInSeconds < 1 || - maximumDurationInSeconds > MAXIMUM_DURATION_IN_SECONDS - ) { - throw new Error( - `MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, - ); - } - - return maximumDurationInSeconds; -} - export function loadMicrovmProviderConfig(): MicrovmProviderConfig { const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); @@ -91,7 +70,6 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { 'MICROVM_EGRESS_NETWORK_CONNECTORS', process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, ), - maximumDurationInSeconds: parseMaximumDuration(process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS), metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts new file mode 100644 index 0000000000..09b6a46f8d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts @@ -0,0 +1 @@ +export const MICROVM_LIFETIME_IN_SECONDS = 28_800; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index cd4fe86250..142adf1695 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -39,7 +39,6 @@ const config: MicrovmProviderConfig = { imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', egressNetworkConnectors: ['arn:egress'], - maximumDurationInSeconds: 1200, metadataSsmPath, logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; @@ -64,6 +63,7 @@ beforeEach(() => { mockMicrovmClient.reset(); vi.clearAllMocks(); vi.useRealTimers(); + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; process.env.AWS_REGION = 'eu-west-1'; process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(); @@ -73,7 +73,8 @@ beforeEach(() => { }); describe('runMicrovmRunner', () => { - it('launches a runner and records durable ownership metadata', async () => { + it('launches a runner for the fixed lifetime and records durable ownership metadata', async () => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn }); await expect( @@ -92,7 +93,7 @@ describe('runMicrovmRunner', () => { imageVersion: '3.0', executionRoleArn: config.executionRoleArn, egressNetworkConnectors: ['arn:egress'], - maximumDurationInSeconds: 1200, + maximumDurationInSeconds: 28_800, logging: config.logging, runHookPayload: '{"version":1}', clientToken: expect.any(String), @@ -105,7 +106,6 @@ describe('runMicrovmRunner', () => { source: 'scale-up-lambda', imageArn, imageVersion: '3.0', - maximumDurationInSeconds: 1200, }); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index 3769d487c7..fb61111d4f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -11,6 +11,7 @@ import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk import type { LambdaRunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; import { createMicrovmRunnerMetadata, deleteMicrovmRunnerMetadata, @@ -77,7 +78,7 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - maximumDurationInSeconds: 1200, metadataSsmPath, }); vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); @@ -93,7 +92,6 @@ describe('createMicrovmRunners', () => { vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - maximumDurationInSeconds: 1200, metadataSsmPath: '/github-action-runners/unit-test/token/metadata', }); @@ -138,7 +136,7 @@ describe('createMicrovmRunners', () => { expect(setMicrovmGithubRunnerId).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', 'github-mvm-1'); }); - it('applies dynamic labels without overriding the deployment-controlled duration', async () => { + it('applies supported dynamic labels to the provider configuration', async () => { const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; const overrideEgressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; @@ -159,7 +157,6 @@ describe('createMicrovmRunners', () => { imageIdentifier: overrideImageArn, imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - maximumDurationInSeconds: 1200, metadataSsmPath, }, environment: 'unit-test', diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts index 10a1fa9349..de7c37692c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -80,12 +80,11 @@ describe('MicroVM metadata lifecycle', () => { source: 'scale-up-lambda', imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', imageVersion: '3.0', - maximumDurationInSeconds: 1200, }); expect(putParameter).toHaveBeenCalledWith( `${metadataSsmPath}/mvm-1`, - JSON.stringify(metadata({ expiresAt: '2026-08-19T10:25:00.000Z' })), + JSON.stringify(metadata({ expiresAt: '2026-08-19T18:05:00.000Z' })), false, ); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index 965b0222d8..f6ef69f390 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -3,6 +3,7 @@ import { deleteParameter, getParametersByPath, putParameter } from '@aws-github- import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; import type { LambdaRunnerSource, RunnerType } from '../../../../core'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; const logger = createChildLogger('microvm-runner-metadata'); @@ -40,7 +41,6 @@ export interface CreateMicrovmRunnerMetadataInput { environment: string; imageArn: string; imageVersion?: string; - maximumDurationInSeconds: number; microvmId: string; runnerOwner: string; runnerType: RunnerType; @@ -166,7 +166,7 @@ export async function createMicrovmRunnerMetadata( imageVersion: input.imageVersion, createdAt: createdAt.toISOString(), expiresAt: new Date( - createdAt.getTime() + (input.maximumDurationInSeconds + EXPIRATION_GRACE_IN_SECONDS) * 1000, + createdAt.getTime() + (MICROVM_LIFETIME_IN_SECONDS + EXPIRATION_GRACE_IN_SECONDS) * 1000, ).toISOString(), }; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index fce2d06137..02818fb939 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -18,7 +18,6 @@ const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const providerConfig = { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - maximumDurationInSeconds: 1200, metadataSsmPath, }; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 06668b935f..58cf080e5e 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -9,7 +9,6 @@ declare global { MICROVM_IMAGE_VERSION: string | undefined; MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; MICROVM_LOG_GROUP: string | undefined; - MICROVM_MAXIMUM_DURATION_IN_SECONDS: string | undefined; MICROVM_METADATA_SSM_PATH: string; } } From 2b086a82342eec1c567533057f5a88bef07d6ec9 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 21 Aug 2026 11:58:33 +0200 Subject: [PATCH 18/21] feat(microvm): tag runner metadata --- .../src/scale-runners/github-runner.ts | 19 ++- .../src/scale-runners/scale-up.test.ts | 19 +++ lambdas/libs/aws-ssm-util/src/index.test.ts | 26 +++ lambdas/libs/aws-ssm-util/src/index.ts | 13 ++ .../compute-providers/aws/microvm/README.md | 47 ++++-- .../microvm/src/control-plane/config.test.ts | 23 +++ .../aws/microvm/src/control-plane/config.ts | 42 +++++ .../src/control-plane/microvms.test.ts | 33 +++- .../aws/microvm/src/control-plane/microvms.ts | 31 +++- .../src/control-plane/runner-config.test.ts | 29 +++- .../src/control-plane/runner-config.ts | 5 +- .../src/control-plane/runner-metadata.test.ts | 125 +++++++++++++- .../src/control-plane/runner-metadata.ts | 155 +++++++++++++++++- .../src/control-plane/scale-down.test.ts | 1 + .../aws/microvm/src/environment.d.ts | 1 + 15 files changed, 523 insertions(+), 46 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 745c66770a..c4ee56d5cc 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -264,6 +264,15 @@ function addDelay(runnerIds: string[]) { return { isDelay, delay }; } +function mergeSsmParameterTags( + configuredTags: CreateGitHubRunnerConfig['ssmParameterStoreTags'], + providerTags: CreateGitHubRunnerConfig['ssmParameterStoreTags'], +): CreateGitHubRunnerConfig['ssmParameterStoreTags'] { + const tagsByKey = new Map(configuredTags.map(({ Key, Value }) => [Key, Value])); + for (const { Key, Value } of providerTags) tagsByKey.set(Key, Value); + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + /** * Creates registration token configuration for non-ephemeral runners. * @@ -285,7 +294,10 @@ async function createRegistrationTokenConfig( for (const runnerId of runnerIds) { await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + tags: mergeSsmParameterTags( + githubRunnerConfig.ssmParameterStoreTags, + options.getSsmParameterTags?.(runnerId) ?? [], + ), }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit @@ -352,7 +364,10 @@ async function createJitConfig( instance: runnerId, }); await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + tags: mergeSsmParameterTags( + githubRunnerConfig.ssmParameterStoreTags, + options.getSsmParameterTags?.(runnerId) ?? [], + ), }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 9df79ceac1..803eb6a659 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -409,6 +409,25 @@ describe('scaleUp with GHES', () => { }); }); + it.each([true, false])( + 'keeps the provider runner identity tag authoritative for ephemeral=%s', + async (ephemeral) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = String(ephemeral); + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify([ + { Key: 'RunnerId', Value: 'configured-value-cannot-win' }, + { Key: 'CostCenter', Value: '1234' }, + ]); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient.commandCalls(PutParameterCommand)[0].args[0].input.Tags).toEqual([ + { Key: 'RunnerId', Value: 'i-12345' }, + { Key: 'CostCenter', Value: '1234' }, + ]); + }, + ); + it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; process.env.RUNNERS_MAXIMUM_COUNT = '2'; diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index ad68c12279..2f6080bb8a 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -1,4 +1,5 @@ import { + AddTagsToResourceCommand, DeleteParameterCommand, GetParameterCommand, GetParameterCommandOutput, @@ -13,6 +14,7 @@ import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; import { + addParameterTags, deleteParameter, getParameter, getParameters, @@ -329,6 +331,30 @@ describe('Test direct parameter path operations', () => { expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: '/metadata/one' }); }); + + it('adds tags to an exact parameter name', async () => { + mockSSMClient.on(AddTagsToResourceCommand).resolves({}); + + await addParameterTags('/metadata/one', [{ Key: 'ghr:environment', Value: 'unit-test' }]); + + expect(mockSSMClient).toHaveReceivedCommandWith(AddTagsToResourceCommand, { + ResourceType: 'Parameter', + ResourceId: '/metadata/one', + Tags: [{ Key: 'ghr:environment', Value: 'unit-test' }], + }); + }); + + it('does not call SSM when there are no parameter tags to add', async () => { + await addParameterTags('/metadata/one', []); + + expect(mockSSMClient).not.toHaveReceivedCommand(AddTagsToResourceCommand); + }); + + it('propagates failures when adding parameter tags', async () => { + mockSSMClient.on(AddTagsToResourceCommand).rejects(new Error('AccessDenied')); + + await expect(addParameterTags('/metadata/one', [{ Key: 'Name', Value: 'runner' }])).rejects.toThrow('AccessDenied'); + }); }); describe('SSM client configuration', () => { diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 9fef6c7b97..ad448b57ac 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -1,4 +1,5 @@ import { + AddTagsToResourceCommand, DeleteParameterCommand, GetParametersByPathCommand, GetParametersCommand, @@ -146,6 +147,18 @@ export async function deleteParameter(parameter_name: string): Promise { await ssmClient().send(new DeleteParameterCommand({ Name: parameter_name })); } +export async function addParameterTags(parameter_name: string, tags: Tag[]): Promise { + if (tags.length === 0) return; + + await ssmClient().send( + new AddTagsToResourceCommand({ + ResourceType: 'Parameter', + ResourceId: parameter_name, + Tags: tags, + }), + ); +} + export const SSM_ADVANCED_TIER_THRESHOLD = 4000; type PutParameterOptions = { overwrite: true; tags?: never } | { overwrite?: false | undefined; tags?: Tag[] }; diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index b2486b695f..1f70ec75e4 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -11,7 +11,7 @@ The MicroVM image `/run` hook receives this `runHookPayload`: } ``` -Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and terminate the MicroVM after the job completes. +Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. Runner ownership and lifecycle state are stored separately as non-secret `String` parameters under `/`. The immutable base @@ -22,6 +22,19 @@ overlap the JIT path, and do not grant the MicroVM execution role access to it. The control plane retries pending cleanup, removes metadata after termination, and reconciles expired records during inventory. +The immutable base metadata parameter is also the canonical tag surface for a +runner. It merges `SSM_PARAMETER_STORE_TAGS` with the Terraform-generated +`MICROVM_METADATA_TAGS`. Terraform supplies `Name`, `ghr:environment`, +`ghr:ssm_config_path`, and `ghr:runner_name_prefix`; the Lambda then adds +authoritative runtime tags: +`ghr:Application`, `ghr:created_by`, `ghr:environment`, `ghr:Owner`, +`ghr:Type`, `ghr:microvm_id`, `ghr:microvm_image_arn`, and, when available, +`ghr:microvm_image_version`. After JIT registration, the control plane adds +`ghr:github_runner_id` and base64url-encoded runner-label groups under +`ghr:runner_labels` through `ghr:runner_labels:5`. Runtime-owned values override +configured collisions. The `aws:` tag prefix is reserved and cannot be used for +these SSM parameters. + The control-plane Lambda requires these provider environment variables: - `MICROVM_IMAGE_ARN` @@ -30,31 +43,33 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) +- `MICROVM_METADATA_TAGS` (optional JSON array of base tags for the canonical metadata parameter) - `MICROVM_LOG_GROUP` (optional) Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). The control-plane role requires `ssm:GetParametersByPath`, `ssm:PutParameter`, -and `ssm:DeleteParameter` on the dedicated metadata prefix, plus -`lambda:ListMicrovms`, `lambda:RunMicrovm`, and `lambda:TerminateMicrovm` for -inventory and lifecycle reconciliation. Restrict `lambda:RunMicrovm` and -`lambda:TerminateMicrovm` to approved image resources; `lambda:ListMicrovms` -does not support resource-level permissions. +`ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the dedicated metadata +prefix, plus `lambda:ListMicrovms`, `lambda:RunMicrovm`, and +`lambda:TerminateMicrovm` for inventory and lifecycle reconciliation. Restrict +`lambda:RunMicrovm` and `lambda:TerminateMicrovm` to approved image resources; +`lambda:ListMicrovms` does not support resource-level permissions. The MicroVM execution role must trust `lambda.amazonaws.com` for both `sts:AssumeRole` and `sts:TagSession`. Restrict `iam:PassRole` to that exact role -with `iam:PassedToService=lambda.amazonaws.com`. Egress connectors also require -`lambda:PassNetworkConnector`; because that action does not currently support -resource-level permissions, enforce the connector boundary with the explicit -dynamic-label allowlist described below. +ARN. Network connectors also require `lambda:PassNetworkConnector`; because +that action does not currently support resource-level permissions, enforce the +connector boundary with the explicit dynamic-label allowlist described below. All MicroVMs using one execution role and JIT prefix share a trust boundary. -Grant that role only `ssm:GetParameter` and `ssm:DeleteParameter` on the JIT -prefix; do not grant parameter-listing APIs or access to the metadata prefix. -The `MicrovmId` tag on each JIT parameter supports operations but is not a -documented binding to the calling MicroVM's session identity. Only allow trusted -images and workloads within a shared role, or isolate trust domains with -separate roles, prefixes, and provider deployments. +Grant that role only `ssm:GetParameter` and `ssm:DeleteParameter` on the +lane-scoped JIT prefix. The image must use the +exact `/` parameter name and must not receive +access to the metadata prefix or path-listing APIs. The `MicrovmId` tag on each +JIT parameter supports operations but is not a documented binding to the +calling MicroVM's session identity. Only allow trusted images and workloads +within a shared role, or isolate trust domains with separate roles, prefixes, +and provider deployments. ## Dynamic labels diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index b68fdffb7c..9646fef794 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -9,6 +9,7 @@ beforeEach(() => { process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + delete process.env.MICROVM_METADATA_TAGS; delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; @@ -24,6 +25,7 @@ describe('loadMicrovmProviderConfig', () => { ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', + metadataTags: [], logging: undefined, }); }); @@ -33,11 +35,19 @@ describe('loadMicrovmProviderConfig', () => { process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + process.env.MICROVM_METADATA_TAGS = JSON.stringify([ + { Key: 'Name', Value: 'unit-test-runner' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + ]); expect(loadMicrovmProviderConfig()).toMatchObject({ imageVersion: '3.0', ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + metadataTags: [ + { Key: 'Name', Value: 'unit-test-runner' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + ], logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }); }); @@ -70,4 +80,17 @@ describe('loadMicrovmProviderConfig', () => { ); }, ); + + it.each([ + '[not-json', + '{}', + '[{"Key":"Name"}]', + '[{"Key":"","Value":"runner"}]', + '[{"Key":"Name","Value":1}]', + '[{"Key":"Name","Value":"one"},{"Key":"Name","Value":"two"}]', + ])('rejects malformed metadata tags %s', (tags) => { + process.env.MICROVM_METADATA_TAGS = tags; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_METADATA_TAGS must/); + }); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index 8eacada06c..bbf26df264 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -1,5 +1,10 @@ import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; +export interface MicrovmMetadataTag { + Key: string; + Value: string; +} + export interface MicrovmProviderConfig { egressNetworkConnectors?: string[]; executionRoleArn: string; @@ -8,6 +13,7 @@ export interface MicrovmProviderConfig { ingressNetworkConnectors?: string[]; logging?: Logging; metadataSsmPath: string; + metadataTags: MicrovmMetadataTag[]; } function requiredEnvironmentValue(name: string, value: string | undefined): string { @@ -55,6 +61,41 @@ function parseNetworkConnectors(name: string, value: string | undefined): string return connectors.map((connector) => connector.trim()); } +function parseMetadataTags(value: string | undefined): MicrovmMetadataTag[] { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return []; + + let tags: unknown; + try { + tags = JSON.parse(configuredValue); + } catch (error) { + throw new Error('MICROVM_METADATA_TAGS must be a JSON array of SSM tag objects', { cause: error }); + } + + if ( + !Array.isArray(tags) || + tags.some( + (tag) => + typeof tag !== 'object' || + tag === null || + !('Key' in tag) || + typeof tag.Key !== 'string' || + tag.Key.length === 0 || + !('Value' in tag) || + typeof tag.Value !== 'string', + ) + ) { + throw new Error('MICROVM_METADATA_TAGS must be a JSON array of SSM tag objects'); + } + + const typedTags = tags as MicrovmMetadataTag[]; + if (new Set(typedTags.map((tag) => tag.Key)).size !== typedTags.length) { + throw new Error('MICROVM_METADATA_TAGS must not contain duplicate tag keys'); + } + + return typedTags; +} + export function loadMicrovmProviderConfig(): MicrovmProviderConfig { const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); @@ -71,6 +112,7 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, ), metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), + metadataTags: parseMetadataTags(process.env.MICROVM_METADATA_TAGS), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index 142adf1695..b90ee2f092 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -24,7 +24,8 @@ import { type MicrovmRunnerMetadata, } from './runner-metadata'; -vi.mock('./runner-metadata', () => ({ +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), createMicrovmRunnerMetadata: vi.fn(), deleteMicrovmRunnerMetadata: vi.fn(), listMicrovmRunnerMetadata: vi.fn(), @@ -40,8 +41,10 @@ const config: MicrovmProviderConfig = { executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', egressNetworkConnectors: ['arn:egress'], metadataSsmPath, + metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; +const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { return { @@ -75,7 +78,7 @@ beforeEach(() => { describe('runMicrovmRunner', () => { it('launches a runner for the fixed lifetime and records durable ownership metadata', async () => { process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; - mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn }); + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn, imageVersion: '3.1' }); await expect( runMicrovmRunner({ @@ -84,6 +87,7 @@ describe('runMicrovmRunner', () => { runHookPayload: '{"version":1}', runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags, source: 'scale-up-lambda', }), ).resolves.toBe('mvm-123'); @@ -105,10 +109,30 @@ describe('runMicrovmRunner', () => { runnerType: 'Org', source: 'scale-up-lambda', imageArn, - imageVersion: '3.0', + imageVersion: '3.1', + metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], + ssmParameterStoreTags, }); }); + it('rejects invalid metadata tags before launching a MicroVM', async () => { + await expect( + runMicrovmRunner({ + config: { + ...config, + metadataTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], + }, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + expect(mockMicrovmClient).not.toHaveReceivedCommand(RunMicrovmCommand); + }); + it('rejects a launch response without an ID', async () => { mockMicrovmClient.on(RunMicrovmCommand).resolves({}); @@ -119,6 +143,7 @@ describe('runMicrovmRunner', () => { runHookPayload: '{}', runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags: [], source: 'pool-lambda', }), ).rejects.toThrow('RunMicrovm returned no microvmId'); @@ -136,6 +161,7 @@ describe('runMicrovmRunner', () => { runHookPayload: '{}', runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags: [], source: 'scale-up-lambda', }), ).rejects.toThrow('metadata failed'); @@ -157,6 +183,7 @@ describe('runMicrovmRunner', () => { runHookPayload: '{}', runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags: [], source: 'scale-up-lambda', }), ).rejects.toThrow('metadata failed'); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index fb61111d4f..58151698ff 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -9,10 +9,17 @@ import { } from '@aws-sdk/client-lambda-microvms'; import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; -import type { LambdaRunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; +import type { + CreateGitHubRunnerConfig, + LambdaRunnerSource, + ListRunnerFilters, + RunnerInfo, + RunnerType, +} from '../../../../core'; import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; import { + assertValidMicrovmMetadataTags, createMicrovmRunnerMetadata, deleteMicrovmRunnerMetadata, listMicrovmRunnerMetadata, @@ -34,6 +41,7 @@ export interface RunMicrovmRunnerInput { runHookPayload: string; runnerOwner: string; runnerType: RunnerType; + ssmParameterStoreTags: CreateGitHubRunnerConfig['ssmParameterStoreTags']; source: LambdaRunnerSource; } @@ -72,6 +80,18 @@ function microvmClient(): LambdaMicrovmsClient { } export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + assertValidMicrovmMetadataTags({ + microvmId: 'microvm-validation', + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.config.imageIdentifier, + imageVersion: input.config.imageVersion ?? 'version-validation', + metadataTags: input.config.metadataTags, + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + const commandInput: RunMicrovmCommandInput = { imageIdentifier: input.config.imageIdentifier, imageVersion: input.config.imageVersion, @@ -95,6 +115,9 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise ({ loadMicrovmProviderConfig: vi.fn() })); vi.mock('./microvms', () => ({ @@ -15,13 +15,14 @@ vi.mock('./microvms', () => ({ })); vi.mock('./runner-metadata', async (importOriginal) => ({ ...(await importOriginal()), - setMicrovmGithubRunnerId: vi.fn(), + setMicrovmGithubRunnerMetadata: vi.fn(), })); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const githubClient = {} as Octokit; const createStartRunnerConfig = vi.fn(); +const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { return { @@ -35,7 +36,7 @@ function runnerConfig(overrides: Partial = {}): Create disableAutoUpdate: true, ssmTokenPath: '/github-action-runners/unit-test/token', ssmConfigPath: '/github-action-runners/unit-test/config', - ssmParameterStoreTags: [], + ssmParameterStoreTags, ...overrides, }; } @@ -47,9 +48,10 @@ beforeEach(() => { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, + metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], }); vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); - vi.mocked(setMicrovmGithubRunnerId).mockResolvedValue(); + vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); vi.mocked(terminateMicrovm).mockResolvedValue(); vi.mocked(isRetryableMicrovmError).mockReturnValue(false); createStartRunnerConfig.mockResolvedValue([]); @@ -93,6 +95,7 @@ describe('createMicrovmRunners', () => { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + metadataTags: [], }); await expect( @@ -114,7 +117,10 @@ describe('createMicrovmRunners', () => { it('launches each MicroVM and delivers its JIT configuration', async () => { vi.mocked(runMicrovmRunner).mockResolvedValueOnce('mvm-1').mockResolvedValueOnce('mvm-2'); createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { - await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: `github-${runnerIds[0]}`, runnerLabels: [] }); + await options?.onJitConfigCreated?.(runnerIds[0], { + githubRunnerId: `github-${runnerIds[0]}`, + runnerLabels: ['self-hosted', 'microvm'], + }); return []; }); @@ -128,12 +134,16 @@ describe('createMicrovmRunners', () => { runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags, source: 'pool-lambda', }); expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); const options = createStartRunnerConfig.mock.calls[0][3]; expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); - expect(setMicrovmGithubRunnerId).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', 'github-mvm-1'); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-mvm-1', + runnerLabels: ['self-hosted', 'microvm'], + }); }); it('applies supported dynamic labels to the provider configuration', async () => { @@ -158,14 +168,19 @@ describe('createMicrovmRunners', () => { imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, + metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], }, environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), runnerOwner: 'Codertocat', runnerType: 'Org', + ssmParameterStoreTags, source: 'scale-up-lambda', }); - expect(setMicrovmGithubRunnerId).toHaveBeenCalledWith(metadataSsmPath, 'mvm-1', 'github-mvm-1'); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-mvm-1', + runnerLabels: [], + }); }); it('retries a JIT setup failure even when runner cleanup fails', async () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index 5393a49d85..2a03f75e07 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -10,7 +10,7 @@ import type { import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; import { loadMicrovmProviderConfig } from './config'; import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; -import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerId } from './runner-metadata'; +import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerMetadata } from './runner-metadata'; const logger = createChildLogger('microvm-runner-config'); @@ -69,13 +69,14 @@ export async function createMicrovmRunners( runHookPayload, runnerOwner: githubRunnerConfig.runnerOwner, runnerType: githubRunnerConfig.runnerType, + ssmParameterStoreTags: githubRunnerConfig.ssmParameterStoreTags, source, }); const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], onJitConfigCreated: async (runnerId, metadata) => { - await setMicrovmGithubRunnerId(config.metadataSsmPath, runnerId, metadata.githubRunnerId); + await setMicrovmGithubRunnerMetadata(config.metadataSsmPath, runnerId, metadata); }, }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts index de7c37692c..10c0e4df21 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -1,4 +1,4 @@ -import { deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { addParameterTags, deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -9,12 +9,13 @@ import { listMicrovmRunnerMetadata, markMicrovmCleanupPending, microvmMetadataParameterName, - setMicrovmGithubRunnerId, + setMicrovmGithubRunnerMetadata, setMicrovmOrphan, type MicrovmRunnerMetadata, } from './runner-metadata'; vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + addParameterTags: vi.fn(), deleteParameter: vi.fn(), getParametersByPath: vi.fn(), putParameter: vi.fn(), @@ -46,6 +47,7 @@ beforeEach(() => { vi.clearAllMocks(); vi.useRealTimers(); vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(addParameterTags).mockResolvedValue(); vi.mocked(getParametersByPath).mockResolvedValue(new Map()); vi.mocked(putParameter).mockResolvedValue(); }); @@ -80,15 +82,71 @@ describe('MicroVM metadata lifecycle', () => { source: 'scale-up-lambda', imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', imageVersion: '3.0', + metadataTags: [ + { Key: 'Name', Value: 'unit-test-runner' }, + { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, + { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, + { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, + ], + ssmParameterStoreTags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'ssm-name-cannot-win' }, + { Key: 'ghr:created_by', Value: 'configured-source-cannot-win' }, + ], }); expect(putParameter).toHaveBeenCalledWith( `${metadataSsmPath}/mvm-1`, JSON.stringify(metadata({ expiresAt: '2026-08-19T18:05:00.000Z' })), false, + { + tags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'unit-test-runner' }, + { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, + { Key: 'ghr:Owner', Value: 'Codertocat' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, + { + Key: 'ghr:microvm_image_arn', + Value: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + }, + { Key: 'ghr:microvm_image_version', Value: '3.0' }, + ], + }, ); }); + it('rejects reserved tag keys and preserves room for late GitHub metadata', async () => { + const input = { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + source: 'scale-up-lambda' as const, + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [], + }; + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + metadataTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + metadataTags: Array.from({ length: 37 }, (_, index) => ({ Key: `Custom${index}`, Value: 'value' })), + }), + ).rejects.toThrow('cannot have more than 44 launch tags'); + expect(putParameter).not.toHaveBeenCalled(); + }); + it('loads active metadata with independent state and cleans expired inactive records', async () => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); @@ -141,12 +199,71 @@ describe('MicroVM metadata lifecycle', () => { ); }); - it('updates GitHub and orphan state without a shared read-modify-write record', async () => { - await setMicrovmGithubRunnerId(metadataSsmPath, 'mvm-1', 'github-42'); + it('updates GitHub state and adds late GitHub metadata tags to the base parameter', async () => { + const runnerLabels = ['self-hosted', 'linux', 'env:unit-test']; + await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-42', + runnerLabels, + }); expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { overwrite: true, }); + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('splits encoded runner labels into SSM-safe tag values', async () => { + const runnerLabels = [`label-${'a'.repeat(140)}`, `label-${'b'.repeat(140)}`]; + + await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-42', + runnerLabels, + }); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[0]]), 'utf8').toString('base64url')}`, + }, + { + Key: 'ghr:runner_labels:2', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[1]]), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('keeps the GitHub runner ID tag when a runner label is too large', async () => { + await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-42', + runnerLabels: ['x'.repeat(300)], + }); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + ]); + }); + + it('keeps the durable GitHub runner ID when late metadata tagging fails', async () => { + vi.mocked(addParameterTags).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { + githubRunnerId: 'github-42', + runnerLabels: [], + }), + ).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + }); + it('updates orphan state without a shared read-modify-write record', async () => { await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { overwrite: true, diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index f6ef69f390..c1cabff703 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -1,8 +1,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { addParameterTags, deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; -import type { LambdaRunnerSource, RunnerType } from '../../../../core'; +import type { GitHubRunnerMetadata, LambdaRunnerSource, RunnerType } from '../../../../core'; +import type { MicrovmMetadataTag } from './config'; import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; const logger = createChildLogger('microvm-runner-metadata'); @@ -10,6 +11,12 @@ const logger = createChildLogger('microvm-runner-metadata'); const METADATA_VERSION = 1; const EXPIRATION_GRACE_IN_SECONDS = 300; const MAX_RECONCILED_RUNNERS = 10; +const MAX_PARAMETER_TAGS = 50; +const MAX_RUNNER_LABEL_TAGS = 5; +const MAX_BASE_PARAMETER_TAGS = MAX_PARAMETER_TAGS - MAX_RUNNER_LABEL_TAGS - 1; +const MAX_TAG_KEY_LENGTH = 128; +const MAX_TAG_VALUE_LENGTH = 256; +const SSM_TAG_VALUE_PATTERN = /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u; const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; const ORPHAN_SUFFIX = '.orphan'; @@ -41,12 +48,127 @@ export interface CreateMicrovmRunnerMetadataInput { environment: string; imageArn: string; imageVersion?: string; + metadataTags: MicrovmMetadataTag[]; microvmId: string; runnerOwner: string; runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; source: LambdaRunnerSource; } +function isProviderOwnedLateTag(key: string): boolean { + return key === 'ghr:github_runner_id' || key === 'ghr:runner_labels' || key.startsWith('ghr:runner_labels:'); +} + +function assertValidParameterTags(tags: MicrovmMetadataTag[]): void { + if (tags.length > MAX_PARAMETER_TAGS) { + throw new Error(`MicroVM metadata cannot have more than ${MAX_PARAMETER_TAGS} tags`); + } + + for (const tag of tags) { + if ( + Array.from(tag.Key).length === 0 || + Array.from(tag.Key).length > MAX_TAG_KEY_LENGTH || + Array.from(tag.Value).length > MAX_TAG_VALUE_LENGTH || + !SSM_TAG_VALUE_PATTERN.test(tag.Key) || + !SSM_TAG_VALUE_PATTERN.test(tag.Value) + ) { + throw new Error(`MicroVM metadata tag '${tag.Key}' does not satisfy SSM tag constraints`); + } + if (tag.Key.toLowerCase().startsWith('aws:')) { + throw new Error(`MicroVM metadata tag '${tag.Key}' uses the AWS-reserved tag prefix`); + } + } +} + +function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadataTag[] { + const tagsByKey = new Map(); + for (const tags of tagSets) { + for (const tag of tags) tagsByKey.set(tag.Key, tag.Value); + } + + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + +function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { + const configuredTags = mergeParameterTags(input.ssmParameterStoreTags, input.metadataTags).filter( + (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version', + ); + const providerTags: MicrovmMetadataTag[] = [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: input.source }, + { Key: 'ghr:environment', Value: input.environment }, + { Key: 'ghr:Owner', Value: input.runnerOwner }, + { Key: 'ghr:Type', Value: input.runnerType }, + { Key: 'ghr:microvm_id', Value: input.microvmId }, + { Key: 'ghr:microvm_image_arn', Value: input.imageArn }, + ]; + if (input.imageVersion !== undefined) { + providerTags.push({ Key: 'ghr:microvm_image_version', Value: input.imageVersion }); + } + + const tags = mergeParameterTags(configuredTags, providerTags); + assertValidParameterTags(tags); + if (tags.length > MAX_BASE_PARAMETER_TAGS) { + throw new Error( + `MicroVM metadata cannot have more than ${MAX_BASE_PARAMETER_TAGS} launch tags because ${MAX_RUNNER_LABEL_TAGS + 1} tags are reserved for GitHub runner metadata`, + ); + } + return tags; +} + +export function assertValidMicrovmMetadataTags(input: CreateMicrovmRunnerMetadataInput): void { + createMetadataParameterTags(input); +} + +function encodeRunnerLabelGroups(labels: string[]): string[] { + const encodedGroups: string[] = []; + let group: string[] = []; + const encode = (values: string[]) => `base64url:${Buffer.from(JSON.stringify(values), 'utf8').toString('base64url')}`; + + for (const label of labels) { + const candidate = [...group, label]; + if (Array.from(encode(candidate)).length <= MAX_TAG_VALUE_LENGTH) { + group = candidate; + continue; + } + if (group.length === 0) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + continue; + } + encodedGroups.push(encode(group)); + group = [label]; + if (Array.from(encode(group)).length > MAX_TAG_VALUE_LENGTH) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + group = []; + } + } + if (group.length > 0) encodedGroups.push(encode(group)); + + if (encodedGroups.length > MAX_RUNNER_LABEL_TAGS) { + logger.warn('GitHub runner label SSM tags were truncated to avoid exceeding the metadata tag budget', { + maxRunnerLabelsTagCount: MAX_RUNNER_LABEL_TAGS, + }); + } + return encodedGroups.slice(0, MAX_RUNNER_LABEL_TAGS); +} + +function createGitHubRunnerMetadataTags(metadata: GitHubRunnerMetadata): MicrovmMetadataTag[] { + const tags: MicrovmMetadataTag[] = [{ Key: 'ghr:github_runner_id', Value: metadata.githubRunnerId }]; + tags.push( + ...encodeRunnerLabelGroups(metadata.runnerLabels).map((Value, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value, + })), + ); + assertValidParameterTags(tags); + return tags; +} + function normalizedPath(path: string): string { return path.trim().replace(/\/+$/, ''); } @@ -170,7 +292,9 @@ export async function createMicrovmRunnerMetadata( ).toISOString(), }; - await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false); + await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false, { + tags: createMetadataParameterTags(input), + }); } function invalidStateReason(parameters: Map, baseName: string): string | undefined { @@ -318,15 +442,28 @@ export async function listMicrovmRunnerMetadata( }; } -export async function setMicrovmGithubRunnerId( +export async function setMicrovmGithubRunnerMetadata( metadataSsmPath: string, microvmId: string, - githubRunnerId: string, + metadata: GitHubRunnerMetadata, ): Promise { - if (!githubRunnerId) throw new Error('GitHub runner ID must not be empty'); - await putParameter(stateParameterName(metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), githubRunnerId, false, { - overwrite: true, - }); + if (!metadata.githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + await putParameter( + stateParameterName(metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), + metadata.githubRunnerId, + false, + { + overwrite: true, + }, + ); + try { + await addParameterTags( + microvmMetadataParameterName(metadataSsmPath, microvmId), + createGitHubRunnerMetadataTags(metadata), + ); + } catch (error) { + logger.error(`Failed to tag MicroVM runner '${microvmId}' with GitHub runner metadata`, { error }); + } } export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: string, orphan: boolean): Promise { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index 02818fb939..9b5df7cd36 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -19,6 +19,7 @@ const providerConfig = { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, + metadataTags: [], }; beforeEach(() => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 58cf080e5e..810e7f3e44 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -10,6 +10,7 @@ declare global { MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; MICROVM_LOG_GROUP: string | undefined; MICROVM_METADATA_SSM_PATH: string; + MICROVM_METADATA_TAGS: string | undefined; } } } From 7b9f2e7f3299bab49f2e1df4a5bfde00a9fd33e9 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 21 Aug 2026 14:27:02 +0200 Subject: [PATCH 19/21] feat(microvm): add runner config ARN to hook payload --- .../compute-providers/aws/microvm/README.md | 25 ++++++++------- .../microvm/src/control-plane/config.test.ts | 19 +++++++++++ .../aws/microvm/src/control-plane/config.ts | 14 ++++++++ .../src/control-plane/microvms.test.ts | 1 + .../src/control-plane/runner-config.test.ts | 32 +++++++++++++++---- .../src/control-plane/runner-config.ts | 14 +++++--- .../src/control-plane/runner-metadata.ts | 12 +++---- .../src/control-plane/scale-down.test.ts | 1 + .../aws/microvm/src/environment.d.ts | 1 + 9 files changed, 91 insertions(+), 28 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index 1f70ec75e4..1cf33df790 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -7,11 +7,12 @@ The MicroVM image `/run` hook receives this `runHookPayload`: ```json { "version": 1, - "runnerConfigSsmPath": "/github-action-runners/example/token" + "runnerConfigSsmArn": "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/example/runners/config", + "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" } ``` -Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. +Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. `runnerConfigSsmArn` is the configuration ARN prefix; the image reads its own non-secret metadata at `/microvm-metadata/`. Neither identifier contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. Runner ownership and lifecycle state are stored separately as non-secret `String` parameters under `/`. The immutable base @@ -44,6 +45,7 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) - `MICROVM_METADATA_TAGS` (optional JSON array of base tags for the canonical metadata parameter) +- `MICROVM_RUNNER_CONFIG_SSM_ARN` (runner configuration SSM ARN prefix passed to the image hook) - `MICROVM_LOG_GROUP` (optional) Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). @@ -61,15 +63,16 @@ ARN. Network connectors also require `lambda:PassNetworkConnector`; because that action does not currently support resource-level permissions, enforce the connector boundary with the explicit dynamic-label allowlist described below. -All MicroVMs using one execution role and JIT prefix share a trust boundary. -Grant that role only `ssm:GetParameter` and `ssm:DeleteParameter` on the -lane-scoped JIT prefix. The image must use the -exact `/` parameter name and must not receive -access to the metadata prefix or path-listing APIs. The `MicrovmId` tag on each -JIT parameter supports operations but is not a documented binding to the -calling MicroVM's session identity. Only allow trusted images and workloads -within a shared role, or isolate trust domains with separate roles, prefixes, -and provider deployments. +All MicroVMs using one execution role, JIT prefix, and metadata prefix share a +trust boundary. Grant that role only `ssm:GetParameter` on +`/microvm-metadata/*`, `ssm:GetParameter` and +`ssm:DeleteParameter` on the lane-scoped JIT prefix, and the runtime log +permissions described above. The image must address its own metadata with its +AWS-provided `microvmId` and must not receive path-listing access. IAM cannot +bind that ID to the calling MicroVM session, so a MicroVM can read other +metadata records in the same lane if it learns their IDs. Only allow trusted +images and workloads within a shared role, or isolate trust domains with +separate roles, prefixes, and provider deployments. ## Dynamic labels diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index 9646fef794..e3935dfc3f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -9,6 +9,8 @@ beforeEach(() => { process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + process.env.MICROVM_RUNNER_CONFIG_SSM_ARN = + 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config'; delete process.env.MICROVM_METADATA_TAGS; delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; @@ -26,6 +28,7 @@ describe('loadMicrovmProviderConfig', () => { egressNetworkConnectors: undefined, metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', metadataTags: [], + runnerConfigSsmArn: process.env.MICROVM_RUNNER_CONFIG_SSM_ARN, logging: undefined, }); }); @@ -56,6 +59,7 @@ describe('loadMicrovmProviderConfig', () => { ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], + ['MICROVM_RUNNER_CONFIG_SSM_ARN', 'MICROVM_RUNNER_CONFIG_SSM_ARN'], ])('requires %s', (environmentVariable, expectedName) => { delete process.env[environmentVariable]; @@ -81,6 +85,21 @@ describe('loadMicrovmProviderConfig', () => { }, ); + it.each([ + '/github-action-runners/unit-test/config', + 'arn:aws:ssm:eu-west-1:123456789012:parameter', + 'arn:aws:s3:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', + 'arn:custom:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', + 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners//config', + 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/../config', + ])('rejects malformed runner configuration SSM ARN %s', (runnerConfigSsmArn) => { + process.env.MICROVM_RUNNER_CONFIG_SSM_ARN = runnerConfigSsmArn; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_RUNNER_CONFIG_SSM_ARN must be a valid SSM parameter ARN prefix', + ); + }); + it.each([ '[not-json', '{}', diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index bbf26df264..ae6d65b896 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -14,6 +14,7 @@ export interface MicrovmProviderConfig { logging?: Logging; metadataSsmPath: string; metadataTags: MicrovmMetadataTag[]; + runnerConfigSsmArn: string; } function requiredEnvironmentValue(name: string, value: string | undefined): string { @@ -37,6 +38,18 @@ function parseMetadataSsmPath(value: string | undefined): string { return path; } +function parseRunnerConfigSsmArn(value: string | undefined): string { + const arn = requiredEnvironmentValue('MICROVM_RUNNER_CONFIG_SSM_ARN', value); + if ( + !/^arn:aws(?:-[a-z0-9-]+)?:ssm:[A-Za-z0-9-]+:\d{12}:parameter\/[A-Za-z0-9_.\-/]+$/.test(arn) || + arn.includes('//') || + arn.split('/').includes('..') + ) { + throw new Error('MICROVM_RUNNER_CONFIG_SSM_ARN must be a valid SSM parameter ARN prefix'); + } + return arn; +} + function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { const configuredValue = optionalEnvironmentValue(value); if (!configuredValue) return undefined; @@ -113,6 +126,7 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { ), metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), metadataTags: parseMetadataTags(process.env.MICROVM_METADATA_TAGS), + runnerConfigSsmArn: parseRunnerConfigSsmArn(process.env.MICROVM_RUNNER_CONFIG_SSM_ARN), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index b90ee2f092..f9267bf036 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -42,6 +42,7 @@ const config: MicrovmProviderConfig = { egressNetworkConnectors: ['arn:egress'], metadataSsmPath, metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], + runnerConfigSsmArn: 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts index 5f5c08f312..93a986d6f7 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -20,6 +20,7 @@ vi.mock('./runner-metadata', async (importOriginal) => ({ const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerConfigSsmArn = 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config'; const githubClient = {} as Octokit; const createStartRunnerConfig = vi.fn(); const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; @@ -49,6 +50,7 @@ beforeEach(() => { executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], + runnerConfigSsmArn, }); vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); @@ -58,10 +60,18 @@ beforeEach(() => { }); describe('createMicrovmRunHookPayload', () => { - it('contains only the versioned SSM prefix contract', () => { - expect(JSON.parse(createMicrovmRunHookPayload('/runner/token'))).toEqual({ + it('contains the versioned runner token path and configuration ARN', () => { + expect( + JSON.parse( + createMicrovmRunHookPayload({ + runnerConfigSsmArn, + runnerTokenSsmPath: '/runner/token', + }), + ), + ).toEqual({ version: 1, - runnerConfigSsmPath: '/runner/token', + runnerConfigSsmArn, + runnerTokenSsmPath: '/runner/token', }); }); }); @@ -88,14 +98,17 @@ describe('createMicrovmRunners', () => { 'scale-up-lambda', ), ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); }); - it('rejects a metadata path that overlaps the JIT configuration path', async () => { + it('rejects a metadata path that overlaps the JIT token path', async () => { vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath: '/github-action-runners/unit-test/token/metadata', metadataTags: [], + runnerConfigSsmArn, }); await expect( @@ -131,7 +144,10 @@ describe('createMicrovmRunners', () => { expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { config: expect.objectContaining({ imageIdentifier: imageArn }), environment: 'unit-test', - runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runHookPayload: createMicrovmRunHookPayload({ + runnerConfigSsmArn, + runnerTokenSsmPath: '/github-action-runners/unit-test/token', + }), runnerOwner: 'Codertocat', runnerType: 'Org', ssmParameterStoreTags, @@ -169,9 +185,13 @@ describe('createMicrovmRunners', () => { executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], + runnerConfigSsmArn, }, environment: 'unit-test', - runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runHookPayload: createMicrovmRunHookPayload({ + runnerConfigSsmArn, + runnerTokenSsmPath: '/github-action-runners/unit-test/token', + }), runnerOwner: 'Codertocat', runnerType: 'Org', ssmParameterStoreTags, diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index 2a03f75e07..597f7dd925 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -15,14 +15,16 @@ import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerMetadata } fr const logger = createChildLogger('microvm-runner-config'); export interface MicrovmRunHookPayloadV1 { - runnerConfigSsmPath: string; + runnerConfigSsmArn: string; + runnerTokenSsmPath: string; version: 1; } -export function createMicrovmRunHookPayload(ssmTokenPath: string): string { +export function createMicrovmRunHookPayload(paths: Omit): string { return JSON.stringify({ version: 1, - runnerConfigSsmPath: ssmTokenPath, + runnerConfigSsmArn: paths.runnerConfigSsmArn, + runnerTokenSsmPath: paths.runnerTokenSsmPath, } satisfies MicrovmRunHookPayloadV1); } @@ -43,7 +45,6 @@ export async function createMicrovmRunners( logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; } - let config; try { config = { ...loadMicrovmProviderConfig(), ...overrides }; @@ -58,7 +59,10 @@ export async function createMicrovmRunners( retryableErrorCount: 0, nonRetryableErrorCount: 0, }; - const runHookPayload = createMicrovmRunHookPayload(githubRunnerConfig.ssmTokenPath); + const runHookPayload = createMicrovmRunHookPayload({ + runnerConfigSsmArn: config.runnerConfigSsmArn, + runnerTokenSsmPath: githubRunnerConfig.ssmTokenPath, + }); for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { let microvmId: string | undefined; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index c1cabff703..30e8bf7f2b 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -194,15 +194,15 @@ function metadataParameterNames(metadataSsmPath: string, microvmId: string): str ]; } -export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerConfigSsmPath: string): void { +export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerTokenSsmPath: string): void { const metadataPath = normalizedPath(metadataSsmPath); - const runnerConfigPath = normalizedPath(runnerConfigSsmPath); + const runnerTokenPath = normalizedPath(runnerTokenSsmPath); if ( - metadataPath === runnerConfigPath || - metadataPath.startsWith(`${runnerConfigPath}/`) || - runnerConfigPath.startsWith(`${metadataPath}/`) + metadataPath === runnerTokenPath || + metadataPath.startsWith(`${runnerTokenPath}/`) || + runnerTokenPath.startsWith(`${metadataPath}/`) ) { - throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT configuration path'); + throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT token path'); } } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index 9b5df7cd36..eceb259f5e 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -20,6 +20,7 @@ const providerConfig = { executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, metadataTags: [], + runnerConfigSsmArn: 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', }; beforeEach(() => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 810e7f3e44..5eab57144c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -11,6 +11,7 @@ declare global { MICROVM_LOG_GROUP: string | undefined; MICROVM_METADATA_SSM_PATH: string; MICROVM_METADATA_TAGS: string | undefined; + MICROVM_RUNNER_CONFIG_SSM_ARN: string; } } } From 00e06ab239ad0b3d5f084eeff46670d05168605d Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 21 Aug 2026 15:12:03 +0200 Subject: [PATCH 20/21] fix(microvm): reuse runner configuration path --- .../compute-providers/aws/microvm/README.md | 25 ++++----- .../microvm/src/control-plane/config.test.ts | 42 -------------- .../aws/microvm/src/control-plane/config.ts | 56 ------------------- .../src/control-plane/microvms.test.ts | 10 +--- .../aws/microvm/src/control-plane/microvms.ts | 2 - .../src/control-plane/runner-config.test.ts | 50 ++++++++++++----- .../src/control-plane/runner-config.ts | 34 +++++++++-- .../src/control-plane/runner-metadata.test.ts | 28 ++++++---- .../src/control-plane/runner-metadata.ts | 9 ++- .../src/control-plane/scale-down.test.ts | 2 - .../aws/microvm/src/environment.d.ts | 2 - 11 files changed, 98 insertions(+), 162 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index 1cf33df790..60099dd19c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -7,27 +7,28 @@ The MicroVM image `/run` hook receives this `runHookPayload`: ```json { "version": 1, - "runnerConfigSsmArn": "arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/example/runners/config", + "runnerConfigSsmPath": "/github-action-runners/example/runners/config", "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" } ``` -Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. `runnerConfigSsmArn` is the configuration ARN prefix; the image reads its own non-secret metadata at `/microvm-metadata/`. Neither identifier contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. +Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image reads its own non-secret metadata at `/microvm-metadata/`. Neither path contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. Runner ownership and lifecycle state are stored separately as non-secret `String` parameters under `/`. The immutable base record and independent state parameters prevent concurrent GitHub ID, orphan, and cleanup updates from overwriting one another. Deleting the JIT SecureString does not delete this metadata. Use a dedicated metadata prefix that does not -overlap the JIT path, and do not grant the MicroVM execution role access to it. -The control plane retries pending cleanup, removes metadata after termination, -and reconciles expired records during inventory. +overlap the JIT path, and grant the MicroVM execution role only the exact +value-read access described below, without path-listing permissions. The control +plane retries pending cleanup, removes metadata after termination, and reconciles +expired records during inventory. The immutable base metadata parameter is also the canonical tag surface for a -runner. It merges `SSM_PARAMETER_STORE_TAGS` with the Terraform-generated -`MICROVM_METADATA_TAGS`. Terraform supplies `Name`, `ghr:environment`, -`ghr:ssm_config_path`, and `ghr:runner_name_prefix`; the Lambda then adds -authoritative runtime tags: +runner. It starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives +`ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` from +the existing `ENVIRONMENT`, `SSM_CONFIG_PATH`, and `RUNNER_NAME_PREFIX` +settings. The Lambda then adds authoritative runtime tags: `ghr:Application`, `ghr:created_by`, `ghr:environment`, `ghr:Owner`, `ghr:Type`, `ghr:microvm_id`, `ghr:microvm_image_arn`, and, when available, `ghr:microvm_image_version`. After JIT registration, the control plane adds @@ -44,8 +45,6 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) -- `MICROVM_METADATA_TAGS` (optional JSON array of base tags for the canonical metadata parameter) -- `MICROVM_RUNNER_CONFIG_SSM_ARN` (runner configuration SSM ARN prefix passed to the image hook) - `MICROVM_LOG_GROUP` (optional) Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). @@ -64,8 +63,8 @@ that action does not currently support resource-level permissions, enforce the connector boundary with the explicit dynamic-label allowlist described below. All MicroVMs using one execution role, JIT prefix, and metadata prefix share a -trust boundary. Grant that role only `ssm:GetParameter` on -`/microvm-metadata/*`, `ssm:GetParameter` and +trust boundary. Grant that role only `ssm:GetParameter` on the Parameter Store +ARN corresponding to `/microvm-metadata/*`, `ssm:GetParameter` and `ssm:DeleteParameter` on the lane-scoped JIT prefix, and the runtime log permissions described above. The image must address its own metadata with its AWS-provided `microvmId` and must not receive path-listing access. IAM cannot diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index e3935dfc3f..b68fdffb7c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -9,9 +9,6 @@ beforeEach(() => { process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; - process.env.MICROVM_RUNNER_CONFIG_SSM_ARN = - 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config'; - delete process.env.MICROVM_METADATA_TAGS; delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; @@ -27,8 +24,6 @@ describe('loadMicrovmProviderConfig', () => { ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', - metadataTags: [], - runnerConfigSsmArn: process.env.MICROVM_RUNNER_CONFIG_SSM_ARN, logging: undefined, }); }); @@ -38,19 +33,11 @@ describe('loadMicrovmProviderConfig', () => { process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; - process.env.MICROVM_METADATA_TAGS = JSON.stringify([ - { Key: 'Name', Value: 'unit-test-runner' }, - { Key: 'ghr:environment', Value: 'unit-test' }, - ]); expect(loadMicrovmProviderConfig()).toMatchObject({ imageVersion: '3.0', ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], - metadataTags: [ - { Key: 'Name', Value: 'unit-test-runner' }, - { Key: 'ghr:environment', Value: 'unit-test' }, - ], logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }); }); @@ -59,7 +46,6 @@ describe('loadMicrovmProviderConfig', () => { ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], - ['MICROVM_RUNNER_CONFIG_SSM_ARN', 'MICROVM_RUNNER_CONFIG_SSM_ARN'], ])('requires %s', (environmentVariable, expectedName) => { delete process.env[environmentVariable]; @@ -84,32 +70,4 @@ describe('loadMicrovmProviderConfig', () => { ); }, ); - - it.each([ - '/github-action-runners/unit-test/config', - 'arn:aws:ssm:eu-west-1:123456789012:parameter', - 'arn:aws:s3:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', - 'arn:custom:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', - 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners//config', - 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/../config', - ])('rejects malformed runner configuration SSM ARN %s', (runnerConfigSsmArn) => { - process.env.MICROVM_RUNNER_CONFIG_SSM_ARN = runnerConfigSsmArn; - - expect(() => loadMicrovmProviderConfig()).toThrow( - 'MICROVM_RUNNER_CONFIG_SSM_ARN must be a valid SSM parameter ARN prefix', - ); - }); - - it.each([ - '[not-json', - '{}', - '[{"Key":"Name"}]', - '[{"Key":"","Value":"runner"}]', - '[{"Key":"Name","Value":1}]', - '[{"Key":"Name","Value":"one"},{"Key":"Name","Value":"two"}]', - ])('rejects malformed metadata tags %s', (tags) => { - process.env.MICROVM_METADATA_TAGS = tags; - - expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_METADATA_TAGS must/); - }); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index ae6d65b896..8eacada06c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -1,10 +1,5 @@ import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; -export interface MicrovmMetadataTag { - Key: string; - Value: string; -} - export interface MicrovmProviderConfig { egressNetworkConnectors?: string[]; executionRoleArn: string; @@ -13,8 +8,6 @@ export interface MicrovmProviderConfig { ingressNetworkConnectors?: string[]; logging?: Logging; metadataSsmPath: string; - metadataTags: MicrovmMetadataTag[]; - runnerConfigSsmArn: string; } function requiredEnvironmentValue(name: string, value: string | undefined): string { @@ -38,18 +31,6 @@ function parseMetadataSsmPath(value: string | undefined): string { return path; } -function parseRunnerConfigSsmArn(value: string | undefined): string { - const arn = requiredEnvironmentValue('MICROVM_RUNNER_CONFIG_SSM_ARN', value); - if ( - !/^arn:aws(?:-[a-z0-9-]+)?:ssm:[A-Za-z0-9-]+:\d{12}:parameter\/[A-Za-z0-9_.\-/]+$/.test(arn) || - arn.includes('//') || - arn.split('/').includes('..') - ) { - throw new Error('MICROVM_RUNNER_CONFIG_SSM_ARN must be a valid SSM parameter ARN prefix'); - } - return arn; -} - function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { const configuredValue = optionalEnvironmentValue(value); if (!configuredValue) return undefined; @@ -74,41 +55,6 @@ function parseNetworkConnectors(name: string, value: string | undefined): string return connectors.map((connector) => connector.trim()); } -function parseMetadataTags(value: string | undefined): MicrovmMetadataTag[] { - const configuredValue = optionalEnvironmentValue(value); - if (!configuredValue) return []; - - let tags: unknown; - try { - tags = JSON.parse(configuredValue); - } catch (error) { - throw new Error('MICROVM_METADATA_TAGS must be a JSON array of SSM tag objects', { cause: error }); - } - - if ( - !Array.isArray(tags) || - tags.some( - (tag) => - typeof tag !== 'object' || - tag === null || - !('Key' in tag) || - typeof tag.Key !== 'string' || - tag.Key.length === 0 || - !('Value' in tag) || - typeof tag.Value !== 'string', - ) - ) { - throw new Error('MICROVM_METADATA_TAGS must be a JSON array of SSM tag objects'); - } - - const typedTags = tags as MicrovmMetadataTag[]; - if (new Set(typedTags.map((tag) => tag.Key)).size !== typedTags.length) { - throw new Error('MICROVM_METADATA_TAGS must not contain duplicate tag keys'); - } - - return typedTags; -} - export function loadMicrovmProviderConfig(): MicrovmProviderConfig { const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); @@ -125,8 +71,6 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, ), metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), - metadataTags: parseMetadataTags(process.env.MICROVM_METADATA_TAGS), - runnerConfigSsmArn: parseRunnerConfigSsmArn(process.env.MICROVM_RUNNER_CONFIG_SSM_ARN), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index f9267bf036..a661ba09c4 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -41,8 +41,6 @@ const config: MicrovmProviderConfig = { executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', egressNetworkConnectors: ['arn:egress'], metadataSsmPath, - metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], - runnerConfigSsmArn: 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; @@ -111,7 +109,6 @@ describe('runMicrovmRunner', () => { source: 'scale-up-lambda', imageArn, imageVersion: '3.1', - metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], ssmParameterStoreTags, }); }); @@ -119,15 +116,12 @@ describe('runMicrovmRunner', () => { it('rejects invalid metadata tags before launching a MicroVM', async () => { await expect( runMicrovmRunner({ - config: { - ...config, - metadataTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], - }, + config, environment: 'unit-test', runHookPayload: '{}', runnerOwner: 'Codertocat', runnerType: 'Org', - ssmParameterStoreTags: [], + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], source: 'scale-up-lambda', }), ).rejects.toThrow('AWS-reserved tag prefix'); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index 58151698ff..ecc17fc4ca 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -88,7 +88,6 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise ({ const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; -const runnerConfigSsmArn = 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config'; +const runnerConfigSsmPath = '/github-action-runners/unit-test/config'; const githubClient = {} as Octokit; const createStartRunnerConfig = vi.fn(); -const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; +const ssmParameterStoreTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:environment', Value: 'caller-cannot-override' }, + { Key: 'ghr:runner_name_prefix', Value: 'caller-cannot-override' }, + { Key: 'ghr:ssm_config_path', Value: 'caller-cannot-override' }, +]; +const microvmMetadataTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: runnerConfigSsmPath }, +]; function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { return { @@ -49,8 +61,6 @@ beforeEach(() => { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, - metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], - runnerConfigSsmArn, }); vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); @@ -60,17 +70,17 @@ beforeEach(() => { }); describe('createMicrovmRunHookPayload', () => { - it('contains the versioned runner token path and configuration ARN', () => { + it('contains the versioned runner token and configuration paths', () => { expect( JSON.parse( createMicrovmRunHookPayload({ - runnerConfigSsmArn, + runnerConfigSsmPath, runnerTokenSsmPath: '/runner/token', }), ), ).toEqual({ version: 1, - runnerConfigSsmArn, + runnerConfigSsmPath, runnerTokenSsmPath: '/runner/token', }); }); @@ -102,13 +112,25 @@ describe('createMicrovmRunners', () => { expect(runMicrovmRunner).not.toHaveBeenCalled(); }); + it('requires an SSM config path', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmConfigPath: '' }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + it('rejects a metadata path that overlaps the JIT token path', async () => { vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath: '/github-action-runners/unit-test/token/metadata', - metadataTags: [], - runnerConfigSsmArn, }); await expect( @@ -145,12 +167,12 @@ describe('createMicrovmRunners', () => { config: expect.objectContaining({ imageIdentifier: imageArn }), environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload({ - runnerConfigSsmArn, + runnerConfigSsmPath, runnerTokenSsmPath: '/github-action-runners/unit-test/token', }), runnerOwner: 'Codertocat', runnerType: 'Org', - ssmParameterStoreTags, + ssmParameterStoreTags: microvmMetadataTags, source: 'pool-lambda', }); expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); @@ -184,17 +206,15 @@ describe('createMicrovmRunners', () => { imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, - metadataTags: [{ Key: 'Name', Value: 'unit-test-runner' }], - runnerConfigSsmArn, }, environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload({ - runnerConfigSsmArn, + runnerConfigSsmPath, runnerTokenSsmPath: '/github-action-runners/unit-test/token', }), runnerOwner: 'Codertocat', runnerType: 'Org', - ssmParameterStoreTags, + ssmParameterStoreTags: microvmMetadataTags, source: 'scale-up-lambda', }); expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, 'mvm-1', { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index 597f7dd925..ed6c13bf5b 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -13,9 +13,15 @@ import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './m import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerMetadata } from './runner-metadata'; const logger = createChildLogger('microvm-runner-config'); +const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ + 'Name', + 'ghr:environment', + 'ghr:runner_name_prefix', + 'ghr:ssm_config_path', +]); export interface MicrovmRunHookPayloadV1 { - runnerConfigSsmArn: string; + runnerConfigSsmPath: string; runnerTokenSsmPath: string; version: 1; } @@ -23,11 +29,23 @@ export interface MicrovmRunHookPayloadV1 { export function createMicrovmRunHookPayload(paths: Omit): string { return JSON.stringify({ version: 1, - runnerConfigSsmArn: paths.runnerConfigSsmArn, + runnerConfigSsmPath: paths.runnerConfigSsmPath, runnerTokenSsmPath: paths.runnerTokenSsmPath, } satisfies MicrovmRunHookPayloadV1); } +function createMicrovmMetadataTags( + config: CreateGitHubRunnerConfig, + environment: string, +): CreateGitHubRunnerConfig['ssmParameterStoreTags'] { + return [ + ...config.ssmParameterStoreTags.filter((tag) => !MICROVM_METADATA_CONTEXT_TAG_KEYS.has(tag.Key)), + { Key: 'ghr:environment', Value: environment }, + { Key: 'ghr:runner_name_prefix', Value: config.runnerNamePrefix }, + { Key: 'ghr:ssm_config_path', Value: config.ssmConfigPath }, + ]; +} + export async function createMicrovmRunners( githubRunnerConfig: CreateGitHubRunnerConfig, numberOfRunners: number, @@ -45,6 +63,10 @@ export async function createMicrovmRunners( logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; } + if (!githubRunnerConfig.ssmConfigPath?.trim()) { + logger.error('Lambda MicroVM runners require SSM_CONFIG_PATH to locate runner metadata'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } let config; try { config = { ...loadMicrovmProviderConfig(), ...overrides }; @@ -60,20 +82,22 @@ export async function createMicrovmRunners( nonRetryableErrorCount: 0, }; const runHookPayload = createMicrovmRunHookPayload({ - runnerConfigSsmArn: config.runnerConfigSsmArn, + runnerConfigSsmPath: githubRunnerConfig.ssmConfigPath, runnerTokenSsmPath: githubRunnerConfig.ssmTokenPath, }); + const environment = process.env.ENVIRONMENT; + const metadataTags = createMicrovmMetadataTags(githubRunnerConfig, environment); for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { let microvmId: string | undefined; try { microvmId = await runMicrovmRunner({ config, - environment: process.env.ENVIRONMENT, + environment, runHookPayload, runnerOwner: githubRunnerConfig.runnerOwner, runnerType: githubRunnerConfig.runnerType, - ssmParameterStoreTags: githubRunnerConfig.ssmParameterStoreTags, + ssmParameterStoreTags: metadataTags, source, }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts index 10c0e4df21..cdae1a58a2 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -82,16 +82,16 @@ describe('MicroVM metadata lifecycle', () => { source: 'scale-up-lambda', imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', imageVersion: '3.0', - metadataTags: [ - { Key: 'Name', Value: 'unit-test-runner' }, - { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, - { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, - { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, - ], ssmParameterStoreTags: [ { Key: 'CostCenter', Value: '1234' }, - { Key: 'Name', Value: 'ssm-name-cannot-win' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, { Key: 'ghr:created_by', Value: 'configured-source-cannot-win' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, + { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, ], }); @@ -102,11 +102,12 @@ describe('MicroVM metadata lifecycle', () => { { tags: [ { Key: 'CostCenter', Value: '1234' }, - { Key: 'Name', Value: 'unit-test-runner' }, - { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, { Key: 'ghr:Owner', Value: 'Codertocat' }, - { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, { Key: 'ghr:Type', Value: 'Org' }, { Key: 'ghr:microvm_id', Value: 'mvm-1' }, { @@ -134,14 +135,17 @@ describe('MicroVM metadata lifecycle', () => { await expect( createMicrovmRunnerMetadata(metadataSsmPath, { ...input, - metadataTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], }), ).rejects.toThrow('AWS-reserved tag prefix'); await expect( createMicrovmRunnerMetadata(metadataSsmPath, { ...input, - metadataTags: Array.from({ length: 37 }, (_, index) => ({ Key: `Custom${index}`, Value: 'value' })), + ssmParameterStoreTags: Array.from({ length: 37 }, (_, index) => ({ + Key: `Custom${index}`, + Value: 'value', + })), }), ).rejects.toThrow('cannot have more than 44 launch tags'); expect(putParameter).not.toHaveBeenCalled(); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index 30e8bf7f2b..4f479d04ff 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -2,8 +2,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { addParameterTags, deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; -import type { GitHubRunnerMetadata, LambdaRunnerSource, RunnerType } from '../../../../core'; -import type { MicrovmMetadataTag } from './config'; +import type { CreateGitHubRunnerConfig, GitHubRunnerMetadata, LambdaRunnerSource, RunnerType } from '../../../../core'; import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; const logger = createChildLogger('microvm-runner-metadata'); @@ -22,6 +21,7 @@ const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; const ORPHAN_SUFFIX = '.orphan'; const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); +type MicrovmMetadataTag = CreateGitHubRunnerConfig['ssmParameterStoreTags'][number]; export interface MicrovmRunnerMetadata { bypassRemoval?: boolean; @@ -48,7 +48,6 @@ export interface CreateMicrovmRunnerMetadataInput { environment: string; imageArn: string; imageVersion?: string; - metadataTags: MicrovmMetadataTag[]; microvmId: string; runnerOwner: string; runnerType: RunnerType; @@ -91,8 +90,8 @@ function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadata } function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { - const configuredTags = mergeParameterTags(input.ssmParameterStoreTags, input.metadataTags).filter( - (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version', + const configuredTags = mergeParameterTags(input.ssmParameterStoreTags).filter( + (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version' && tag.Key !== 'Name', ); const providerTags: MicrovmMetadataTag[] = [ { Key: 'ghr:Application', Value: 'github-action-runner' }, diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index eceb259f5e..02818fb939 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -19,8 +19,6 @@ const providerConfig = { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, - metadataTags: [], - runnerConfigSsmArn: 'arn:aws:ssm:eu-west-1:123456789012:parameter/github-action-runners/unit-test/config', }; beforeEach(() => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 5eab57144c..58cf080e5e 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -10,8 +10,6 @@ declare global { MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; MICROVM_LOG_GROUP: string | undefined; MICROVM_METADATA_SSM_PATH: string; - MICROVM_METADATA_TAGS: string | undefined; - MICROVM_RUNNER_CONFIG_SSM_ARN: string; } } } From a63113c202538b206846bcee44181c0489525519 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 21 Aug 2026 19:25:18 +0200 Subject: [PATCH 21/21] feat(microvm): extend runner lifecycle metadata --- .../src/scale-runners/github-runner.test.ts | 72 ++++ .../src/scale-runners/github-runner.ts | 9 +- .../compute-providers/aws/microvm/README.md | 51 ++- .../microvm/src/control-plane/config.test.ts | 14 +- .../aws/microvm/src/control-plane/config.ts | 12 +- .../src/control-plane/microvms.test.ts | 66 ++-- .../aws/microvm/src/control-plane/microvms.ts | 47 ++- .../src/control-plane/runner-config.test.ts | 124 +++++-- .../src/control-plane/runner-config.ts | 74 +++- .../src/control-plane/runner-metadata.test.ts | 322 +++++++++++++++--- .../src/control-plane/runner-metadata.ts | 284 ++++++++++----- .../src/control-plane/scale-down.test.ts | 8 +- .../microvm/src/control-plane/scale-down.ts | 10 +- .../aws/microvm/src/environment.d.ts | 1 + 14 files changed, 849 insertions(+), 245 deletions(-) create mode 100644 lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts new file mode 100644 index 0000000000..4f687b1644 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts @@ -0,0 +1,72 @@ +import { putParameter } from '@aws-github-runner/aws-ssm-util'; +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createStartRunnerConfig } from './github-runner'; +import type { CreateGitHubRunnerConfig } from './types'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +const githubRunnerConfig: CreateGitHubRunnerConfig = { + disableAutoUpdate: true, + enableJitConfig: true, + ephemeral: true, + runnerGroup: 'Default', + runnerLabels: 'self-hosted,linux', + runnerNamePrefix: 'runner-', + runnerOwner: 'octocat/runner', + runnerType: 'Repo', + ssmConfigPath: '/github-action-runners/test/config', + ssmParameterStoreTags: [], + ssmTokenPath: '/github-action-runners/test/tokens', +}; + +const generateRunnerJitconfigForRepo = vi.fn(); +const githubClient = { + actions: { generateRunnerJitconfigForRepo }, +} as unknown as Octokit; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(putParameter).mockResolvedValue(); + generateRunnerJitconfigForRepo.mockResolvedValue({ + data: { + encoded_jit_config: 'encoded-jit-config', + runner: { id: 42 }, + }, + headers: {}, + }); +}); + +describe('createStartRunnerConfig', () => { + it('persists JIT configuration before notifying the provider', async () => { + const onJitConfigCreated = vi.fn(async () => { + expect(putParameter).toHaveBeenCalledWith( + '/github-action-runners/test/tokens/microvm-1', + 'encoded-jit-config', + true, + { tags: [] }, + ); + }); + + await expect( + createStartRunnerConfig(githubRunnerConfig, ['microvm-1'], githubClient, { onJitConfigCreated }), + ).resolves.toEqual([]); + expect(onJitConfigCreated).toHaveBeenCalledWith('microvm-1', { + githubRunnerId: '42', + runnerLabels: ['self-hosted', 'linux'], + }); + }); + + it('reports provider post-write fencing failures while leaving cleanup to the provider', async () => { + const onJitConfigCreated = vi.fn().mockRejectedValue(new Error('cleanup already requested')); + + await expect( + createStartRunnerConfig(githubRunnerConfig, ['microvm-1'], githubClient, { onJitConfigCreated }), + ).resolves.toEqual(['microvm-1']); + expect(putParameter).toHaveBeenCalledOnce(); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index c4ee56d5cc..db7eb2b5ca 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -354,11 +354,6 @@ async function createJitConfig( metricGitHubAppRateLimit(runnerConfig.headers, githubRunnerConfig.appIndex); - await options.onJitConfigCreated?.(runnerId, { - githubRunnerId: runnerConfig.data.runner.id.toString(), - runnerLabels, - }); - // store jit config in ssm parameter store logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, @@ -369,6 +364,10 @@ async function createJitConfig( options.getSsmParameterTags?.(runnerId) ?? [], ), }); + await options.onJitConfigCreated?.(runnerId, { + githubRunnerId: runnerConfig.data.runner.id.toString(), + runnerLabels, + }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit await delay(25); diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md index 60099dd19c..bc672dd2a2 100644 --- a/lambdas/libs/compute-providers/aws/microvm/README.md +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -7,12 +7,14 @@ The MicroVM image `/run` hook receives this `runHookPayload`: ```json { "version": 1, + "imageArn": "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner", + "imageVersion": "12.0", "runnerConfigSsmPath": "/github-action-runners/example/runners/config", "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" } ``` -Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image reads its own non-secret metadata at `/microvm-metadata/`. Neither path contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. +Lambda adds `microvmId` beside that payload. `imageArn` and `imageVersion` are the requested launch values and are included together when an explicit image version is selected. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image separately polls its complete non-secret tag map at `/microvm-metadata/.tags`. The control plane stores the JIT parameter before the provider callback writes the tag map, preventing cleanup from deleting an absent JIT that could otherwise be recreated later. Neither metadata record contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. Runner ownership and lifecycle state are stored separately as non-secret `String` parameters under `/`. The immutable base @@ -24,8 +26,9 @@ value-read access described below, without path-listing permissions. The control plane retries pending cleanup, removes metadata after termination, and reconciles expired records during inventory. -The immutable base metadata parameter is also the canonical tag surface for a -runner. It starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives +The immutable base metadata parameter carries the same AWS resource tags that +are serialized as a JSON object in the `.tags` parameter. The tag +set starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` from the existing `ENVIRONMENT`, `SSM_CONFIG_PATH`, and `RUNNER_NAME_PREFIX` settings. The Lambda then adds authoritative runtime tags: @@ -35,7 +38,30 @@ settings. The Lambda then adds authoritative runtime tags: `ghr:github_runner_id` and base64url-encoded runner-label groups under `ghr:runner_labels` through `ghr:runner_labels:5`. Runtime-owned values override configured collisions. The `aws:` tag prefix is reserved and cannot be used for -these SSM parameters. +these SSM parameters. The `.tags` value may use the Parameter Store advanced +tier when its UTF-8 representation is at least 4,000 bytes and is rejected if +the complete value could exceed the 8 KiB Parameter Store limit. + +Final cleanup deletes `/`, the +`.github-runner-id`, `.orphan`, and `.tags` companions, the base ownership +record, and `.cleanup-requested-at` last. The tombstone keeps its original +timestamp through a five-minute grace window so cleanup can repeatedly revoke a +late JIT write before removing every record. Missing parameters are treated as +already cleaned. + +The runner configuration publishes `/enable_cloudwatch` +and, when enabled, `/cloudwatch_agent_config_runner`. +The generated agent configuration reads these image-owned files by default: + +- `/var/log/microvm/internal-services.log` +- `/var/log/microvm/run.log` +- `/opt/actions-runner/_diag/Runner_**.log` + +Their default log-group suffixes are `internal_service`, `run`, and `runner`, +and `{microvm_id}` is an image-expanded log-stream placeholder. The first two +files are part of the MicroVM image contract; the portable lifecycle hook does +not create CloudWatch-specific files. Native RunMicrovm stdout and stderr stay +enabled independently as the early-startup and failure backstop. The control-plane Lambda requires these provider environment variables: @@ -46,12 +72,14 @@ The control-plane Lambda requires these provider environment variables: - `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) - `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) - `MICROVM_LOG_GROUP` (optional) +- `SSM_TOKEN_PATH` (lane-scoped JIT parameter path) Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). -The control-plane role requires `ssm:GetParametersByPath`, `ssm:PutParameter`, -`ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the dedicated metadata -prefix, plus `lambda:ListMicrovms`, `lambda:RunMicrovm`, and +The control-plane role requires `ssm:GetParametersByPath`, `ssm:GetParameters`, +`ssm:PutParameter`, `ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the +dedicated metadata prefix, plus a separate `ssm:DeleteParameter` grant on the +lane-scoped JIT prefix, and `lambda:ListMicrovms`, `lambda:RunMicrovm`, and `lambda:TerminateMicrovm` for inventory and lifecycle reconciliation. Restrict `lambda:RunMicrovm` and `lambda:TerminateMicrovm` to approved image resources; `lambda:ListMicrovms` does not support resource-level permissions. @@ -64,10 +92,11 @@ connector boundary with the explicit dynamic-label allowlist described below. All MicroVMs using one execution role, JIT prefix, and metadata prefix share a trust boundary. Grant that role only `ssm:GetParameter` on the Parameter Store -ARN corresponding to `/microvm-metadata/*`, `ssm:GetParameter` and -`ssm:DeleteParameter` on the lane-scoped JIT prefix, and the runtime log -permissions described above. The image must address its own metadata with its -AWS-provided `microvmId` and must not receive path-listing access. IAM cannot +ARN corresponding to `/microvm-metadata/*` and the exact +CloudWatch configuration parameters, `ssm:GetParameter` and +`ssm:DeleteParameter` on the lane-scoped JIT prefix, and stream-write access to +the provider-managed log groups. The image must address its own metadata with +its AWS-provided `microvmId` and must not receive path-listing access. IAM cannot bind that ID to the calling MicroVM session, so a MicroVM can read other metadata records in the same lane if it learns their IDs. Only allow trusted images and workloads within a shared role, or isolate trust domains with diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index b68fdffb7c..e58d73093c 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -9,6 +9,7 @@ beforeEach(() => { process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/unit-test/token/'; delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; @@ -24,6 +25,7 @@ describe('loadMicrovmProviderConfig', () => { ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', + runnerTokenSsmPath: '/github-action-runners/unit-test/token', logging: undefined, }); }); @@ -46,6 +48,7 @@ describe('loadMicrovmProviderConfig', () => { ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], + ['SSM_TOKEN_PATH', 'SSM_TOKEN_PATH'], ])('requires %s', (environmentVariable, expectedName) => { delete process.env[environmentVariable]; @@ -60,7 +63,7 @@ describe('loadMicrovmProviderConfig', () => { expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); }); - it.each(['metadata', '/', '/metadata//nested', '/metadata/has space'])( + it.each(['metadata', '/', '/metadata//nested', '/metadata/../nested', '/metadata/has space'])( 'rejects malformed metadata SSM path %s', (metadataPath) => { process.env.MICROVM_METADATA_SSM_PATH = metadataPath; @@ -70,4 +73,13 @@ describe('loadMicrovmProviderConfig', () => { ); }, ); + + it.each(['token', '/', '/token//nested', '/token/../nested', '/token/has space'])( + 'rejects malformed JIT SSM path %s', + (tokenPath) => { + process.env.SSM_TOKEN_PATH = tokenPath; + + expect(() => loadMicrovmProviderConfig()).toThrow('SSM_TOKEN_PATH must be a valid absolute SSM parameter path'); + }, + ); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index 8eacada06c..b86331967b 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -8,6 +8,7 @@ export interface MicrovmProviderConfig { ingressNetworkConnectors?: string[]; logging?: Logging; metadataSsmPath: string; + runnerTokenSsmPath: string; } function requiredEnvironmentValue(name: string, value: string | undefined): string { @@ -23,10 +24,10 @@ function optionalEnvironmentValue(value: string | undefined): string | undefined return trimmed ? trimmed : undefined; } -function parseMetadataSsmPath(value: string | undefined): string { - const path = requiredEnvironmentValue('MICROVM_METADATA_SSM_PATH', value).replace(/\/+$/, ''); - if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//')) { - throw new Error('MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path'); +function parseSsmPath(name: string, value: string | undefined): string { + const path = requiredEnvironmentValue(name, value).replace(/\/+$/, ''); + if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//') || path.split('/').includes('..')) { + throw new Error(`${name} must be a valid absolute SSM parameter path`); } return path; } @@ -70,7 +71,8 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { 'MICROVM_EGRESS_NETWORK_CONNECTORS', process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, ), - metadataSsmPath: parseMetadataSsmPath(process.env.MICROVM_METADATA_SSM_PATH), + metadataSsmPath: parseSsmPath('MICROVM_METADATA_SSM_PATH', process.env.MICROVM_METADATA_SSM_PATH), + runnerTokenSsmPath: parseSsmPath('SSM_TOKEN_PATH', process.env.SSM_TOKEN_PATH), logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts index a661ba09c4..3271fd8b98 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -18,7 +18,7 @@ import { } from './microvms'; import { createMicrovmRunnerMetadata, - deleteMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, listMicrovmRunnerMetadata, markMicrovmCleanupPending, type MicrovmRunnerMetadata, @@ -27,7 +27,7 @@ import { vi.mock('./runner-metadata', async (importOriginal) => ({ ...(await importOriginal()), createMicrovmRunnerMetadata: vi.fn(), - deleteMicrovmRunnerMetadata: vi.fn(), + deleteMicrovmRunnerJitConfig: vi.fn(), listMicrovmRunnerMetadata: vi.fn(), markMicrovmCleanupPending: vi.fn(), })); @@ -35,12 +35,15 @@ vi.mock('./runner-metadata', async (importOriginal) => ({ const mockMicrovmClient = mockClient(LambdaMicrovmsClient); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; const config: MicrovmProviderConfig = { imageIdentifier: imageArn, imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', egressNetworkConnectors: ['arn:egress'], metadataSsmPath, + runnerTokenSsmPath, logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, }; const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; @@ -68,8 +71,8 @@ beforeEach(() => { delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; process.env.AWS_REGION = 'eu-west-1'; process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; - vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(); - vi.mocked(deleteMicrovmRunnerMetadata).mockResolvedValue(); + vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(ssmParameterStoreTags); + vi.mocked(deleteMicrovmRunnerJitConfig).mockResolvedValue(); vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ cleanupMicrovmIds: [], metadataById: new Map() }); vi.mocked(markMicrovmCleanupPending).mockResolvedValue(); }); @@ -89,7 +92,7 @@ describe('runMicrovmRunner', () => { ssmParameterStoreTags, source: 'scale-up-lambda', }), - ).resolves.toBe('mvm-123'); + ).resolves.toEqual({ microvmId: 'mvm-123', metadataTags: ssmParameterStoreTags }); expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { imageIdentifier: imageArn, @@ -216,7 +219,7 @@ describe('listMicrovmRunners', () => { runnerOwner: 'Codertocat', runnerType: 'Org', }, - metadataSsmPath, + ssmPaths, ), ).resolves.toEqual([ { @@ -237,7 +240,7 @@ describe('listMicrovmRunners', () => { nextToken: 'page-2', }); expect(listMicrovmRunnerMetadata).toHaveBeenCalledWith( - metadataSsmPath, + ssmPaths, new Map([ ['mvm-managed', 'RUNNING'], ['mvm-terminated', 'TERMINATED'], @@ -268,10 +271,10 @@ describe('listMicrovmRunners', () => { ]), }); - await expect(listMicrovmRunners({ environment: 'unit-test' }, metadataSsmPath)).resolves.toEqual([]); - await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, metadataSsmPath)).resolves.toEqual([]); - await expect(listMicrovmRunners({ runnerType: 'Org' }, metadataSsmPath)).resolves.toEqual([]); - await expect(listMicrovmRunners({ orphan: true }, metadataSsmPath)).resolves.toEqual([]); + await expect(listMicrovmRunners({ environment: 'unit-test' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true }, ssmPaths)).resolves.toEqual([]); }); it('fails closed for an image mismatch while ignoring unowned MicroVMs', async () => { @@ -288,7 +291,7 @@ describe('listMicrovmRunners', () => { ]), }); - await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('does not match its metadata'); + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('does not match its metadata'); }); it('attempts every pending cleanup and fails inventory closed when a retry fails', async () => { @@ -306,7 +309,7 @@ describe('listMicrovmRunners', () => { metadataById: new Map(), }); - await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('cleanup failed'); + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('cleanup failed'); expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-first', }); @@ -322,7 +325,7 @@ describe('listMicrovmRunners', () => { }); vi.mocked(listMicrovmRunnerMetadata).mockRejectedValue(new Error('AccessDenied')); - await expect(listMicrovmRunners({}, metadataSsmPath)).rejects.toThrow('AccessDenied'); + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('AccessDenied'); }); }); @@ -330,26 +333,47 @@ describe('MicroVM lifecycle helpers', () => { it('retains metadata until inventory observes a terminated MicroVM', async () => { mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); - await terminateMicrovm('mvm-123', metadataSsmPath); + await terminateMicrovm('mvm-123', ssmPaths); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); - expect(deleteMicrovmRunnerMetadata).not.toHaveBeenCalled(); }); - it('treats an already terminated MicroVM as successful cleanup', async () => { + it('retains the tombstone when the MicroVM is already terminated so a late JIT write can be revoked', async () => { const notFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); mockMicrovmClient.on(TerminateMicrovmCommand).rejects(notFound); - await expect(terminateMicrovm('mvm-gone', metadataSsmPath)).resolves.toBeUndefined(); - expect(deleteMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + await expect(terminateMicrovm('mvm-gone', ssmPaths)).resolves.toBeUndefined(); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-gone'); }); it('retains metadata and marks cleanup pending when termination fails', async () => { mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); - await expect(terminateMicrovm('mvm-123', metadataSsmPath)).rejects.toThrow('terminate failed'); + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toThrow('terminate failed'); expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); - expect(deleteMicrovmRunnerMetadata).not.toHaveBeenCalled(); + }); + + it('retains the cleanup marker and reports a JIT deletion failure after termination succeeds', async () => { + const error = new Error('JIT cleanup failed'); + vi.mocked(deleteMicrovmRunnerJitConfig).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('still terminates and reports a cleanup-marker failure for retry', async () => { + const error = new Error('metadata cleanup marker failed'); + vi.mocked(markMicrovmCleanupPending).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); }); it('evaluates the configured boot window', () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index ecc17fc4ca..5ffd6d74b5 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -21,9 +21,10 @@ import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; import { assertValidMicrovmMetadataTags, createMicrovmRunnerMetadata, - deleteMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, listMicrovmRunnerMetadata, markMicrovmCleanupPending, + type MicrovmSsmPaths, } from './runner-metadata'; const logger = createChildLogger('microvm-runners'); @@ -45,6 +46,11 @@ export interface RunMicrovmRunnerInput { source: LambdaRunnerSource; } +export interface RunMicrovmRunnerResult { + metadataTags: CreateGitHubRunnerConfig['ssmParameterStoreTags']; + microvmId: string; +} + interface AwsErrorLike extends Error { cause?: unknown; code?: string; @@ -79,7 +85,7 @@ function microvmClient(): LambdaMicrovmsClient { return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); } -export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { assertValidMicrovmMetadataTags({ microvmId: 'microvm-validation', environment: input.environment, @@ -118,7 +124,7 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + await terminateMicrovm(response.microvmId, input.config).catch((terminationError) => { logger.error(`Failed to terminate untracked MicroVM runner '${response.microvmId}'`, { error: terminationError, }); }); throw error; } - - return response.microvmId; } export async function listMicrovmRunners( filters: ListRunnerFilters = {}, - metadataSsmPath = loadMicrovmProviderConfig().metadataSsmPath, + paths: MicrovmSsmPaths = loadMicrovmProviderConfig(), ): Promise { const client = microvmClient(); const items: MicrovmItem[] = []; @@ -169,13 +174,13 @@ export async function listMicrovmRunners( const microvmStates = new Map( items.flatMap((item) => (item.microvmId && item.state ? [[item.microvmId, item.state] as const] : [])), ); - const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(metadataSsmPath, microvmStates); + const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(paths, microvmStates); let cleanupError: unknown; for (const microvmId of cleanupMicrovmIds) { logger.warn(`Retrying cleanup of MicroVM runner '${microvmId}'`); try { - await terminateMicrovm(microvmId, metadataSsmPath); + await terminateMicrovm(microvmId, paths); } catch (error) { cleanupError ??= error; logger.error(`Failed to retry cleanup of MicroVM runner '${microvmId}'`, { error }); @@ -213,22 +218,34 @@ export async function listMicrovmRunners( return runners; } -export async function terminateMicrovm(microvmId: string, metadataSsmPath: string): Promise { +export async function terminateMicrovm(microvmId: string, paths: MicrovmSsmPaths): Promise { + let cleanupPreparationError: unknown; + try { + await markMicrovmCleanupPending(paths.metadataSsmPath, microvmId); + } catch (error) { + cleanupPreparationError = error; + logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error }); + } + + try { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + } catch (error) { + cleanupPreparationError ??= error; + logger.error(`Failed to delete JIT configuration for MicroVM runner '${microvmId}'`, { error }); + } + try { await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); } catch (error) { if (error instanceof Error && error.name === 'ResourceNotFoundException') { - await deleteMicrovmRunnerMetadata(metadataSsmPath, microvmId); + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; return; } - await markMicrovmCleanupPending(metadataSsmPath, microvmId).catch((metadataError) => { - logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error: metadataError }); - }); throw error; } - await markMicrovmCleanupPending(metadataSsmPath, microvmId); + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; } export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts index 635ec1a82f..586d7f6c84 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -21,6 +21,7 @@ vi.mock('./runner-metadata', async (importOriginal) => ({ const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; const runnerConfigSsmPath = '/github-action-runners/unit-test/config'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; const githubClient = {} as Octokit; const createStartRunnerConfig = vi.fn(); const ssmParameterStoreTags = [ @@ -36,6 +37,18 @@ const microvmMetadataTags = [ { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, { Key: 'ghr:ssm_config_path', Value: runnerConfigSsmPath }, ]; +const canonicalMetadataTags = [ + ...microvmMetadataTags, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; +const providerConfig = { + imageIdentifier: imageArn, + imageVersion: '2.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { return { @@ -47,7 +60,7 @@ function runnerConfig(overrides: Partial = {}): Create runnerOwner: 'Codertocat', runnerType: 'Org', disableAutoUpdate: true, - ssmTokenPath: '/github-action-runners/unit-test/token', + ssmTokenPath: runnerTokenSsmPath, ssmConfigPath: '/github-action-runners/unit-test/config', ssmParameterStoreTags, ...overrides, @@ -57,12 +70,8 @@ function runnerConfig(overrides: Partial = {}): Create beforeEach(() => { vi.clearAllMocks(); process.env.ENVIRONMENT = 'unit-test'; - vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ - imageIdentifier: imageArn, - executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', - metadataSsmPath, - }); - vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(runMicrovmRunner).mockResolvedValue({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }); vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); vi.mocked(terminateMicrovm).mockResolvedValue(); vi.mocked(isRetryableMicrovmError).mockReturnValue(false); @@ -70,20 +79,42 @@ beforeEach(() => { }); describe('createMicrovmRunHookPayload', () => { - it('contains the versioned runner token and configuration paths', () => { + it('contains the image and versioned runner paths', () => { expect( JSON.parse( createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', runnerConfigSsmPath, runnerTokenSsmPath: '/runner/token', }), ), ).toEqual({ + imageArn, + imageVersion: '2.0', version: 1, runnerConfigSsmPath, runnerTokenSsmPath: '/runner/token', }); }); + + it('requires the image ARN and version to be provided together', () => { + expect(() => + createMicrovmRunHookPayload({ + imageArn, + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ).toThrow('MicroVM hook payload image ARN and version must be provided together'); + }); + + it('omits image metadata when no explicit image version is selected', () => { + expect(JSON.parse(createMicrovmRunHookPayload({ runnerConfigSsmPath, runnerTokenSsmPath }))).toEqual({ + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath, + }); + }); }); describe('createMicrovmRunners', () => { @@ -131,6 +162,7 @@ describe('createMicrovmRunners', () => { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + runnerTokenSsmPath, }); await expect( @@ -139,6 +171,36 @@ describe('createMicrovmRunners', () => { expect(runMicrovmRunner).not.toHaveBeenCalled(); }); + it('canonicalizes the configuration and token paths before launching or writing JIT configuration', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmConfigPath: `${runnerConfigSsmPath}/`, ssmTokenPath: `${runnerTokenSsmPath}/` }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: ['mvm-1'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenCalledWith( + expect.objectContaining({ + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ssmParameterStoreTags: microvmMetadataTags, + }), + ); + expect(createStartRunnerConfig).toHaveBeenCalledWith( + expect.objectContaining({ ssmConfigPath: runnerConfigSsmPath, ssmTokenPath: runnerTokenSsmPath }), + ['mvm-1'], + githubClient, + expect.any(Object), + ); + }); + it('classifies invalid provider configuration as non-retryable', async () => { vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { throw new Error('missing image'); @@ -150,7 +212,9 @@ describe('createMicrovmRunners', () => { }); it('launches each MicroVM and delivers its JIT configuration', async () => { - vi.mocked(runMicrovmRunner).mockResolvedValueOnce('mvm-1').mockResolvedValueOnce('mvm-2'); + vi.mocked(runMicrovmRunner) + .mockResolvedValueOnce({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }) + .mockResolvedValueOnce({ microvmId: 'mvm-2', metadataTags: canonicalMetadataTags }); createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: `github-${runnerIds[0]}`, @@ -167,8 +231,10 @@ describe('createMicrovmRunners', () => { config: expect.objectContaining({ imageIdentifier: imageArn }), environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', runnerConfigSsmPath, - runnerTokenSsmPath: '/github-action-runners/unit-test/token', + runnerTokenSsmPath, }), runnerOwner: 'Codertocat', runnerType: 'Org', @@ -178,10 +244,16 @@ describe('createMicrovmRunners', () => { expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); const options = createStartRunnerConfig.mock.calls[0][3]; expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); - expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-mvm-1', - runnerLabels: ['self-hosted', 'microvm'], - }); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith( + 1, + providerConfig, + 'mvm-1', + { + githubRunnerId: 'github-mvm-1', + runnerLabels: ['self-hosted', 'microvm'], + }, + canonicalMetadataTags, + ); }); it('applies supported dynamic labels to the provider configuration', async () => { @@ -206,21 +278,31 @@ describe('createMicrovmRunners', () => { imageVersion: '3.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, + runnerTokenSsmPath, }, environment: 'unit-test', runHookPayload: createMicrovmRunHookPayload({ + imageArn: overrideImageArn, + imageVersion: '3.0', runnerConfigSsmPath, - runnerTokenSsmPath: '/github-action-runners/unit-test/token', + runnerTokenSsmPath, }), runnerOwner: 'Codertocat', runnerType: 'Org', ssmParameterStoreTags: microvmMetadataTags, source: 'scale-up-lambda', }); - expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-mvm-1', - runnerLabels: [], - }); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith( + { + ...providerConfig, + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + 'mvm-1', + { githubRunnerId: 'github-mvm-1', runnerLabels: [] }, + canonicalMetadataTags, + ); }); it('retries a JIT setup failure even when runner cleanup fails', async () => { @@ -231,7 +313,7 @@ describe('createMicrovmRunners', () => { createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); }); it.each([ @@ -254,6 +336,6 @@ describe('createMicrovmRunners', () => { createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); }); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index ed6c13bf5b..15f1fb4502 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -10,7 +10,12 @@ import type { import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; import { loadMicrovmProviderConfig } from './config'; import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; -import { assertSeparatedMicrovmMetadataPath, setMicrovmGithubRunnerMetadata } from './runner-metadata'; +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + normalizeMicrovmSsmPath, + setMicrovmGithubRunnerMetadata, +} from './runner-metadata'; const logger = createChildLogger('microvm-runner-config'); const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ @@ -21,16 +26,30 @@ const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ ]); export interface MicrovmRunHookPayloadV1 { + imageArn?: string; + imageVersion?: string; runnerConfigSsmPath: string; runnerTokenSsmPath: string; version: 1; } -export function createMicrovmRunHookPayload(paths: Omit): string { +export function createMicrovmRunHookPayload(payload: Omit): string { + const hasImageArn = payload.imageArn !== undefined; + const hasImageVersion = payload.imageVersion !== undefined; + if (hasImageArn !== hasImageVersion) { + throw new Error('MicroVM hook payload image ARN and version must be provided together'); + } + return JSON.stringify({ version: 1, - runnerConfigSsmPath: paths.runnerConfigSsmPath, - runnerTokenSsmPath: paths.runnerTokenSsmPath, + ...(hasImageArn + ? { + imageArn: payload.imageArn, + imageVersion: payload.imageVersion, + } + : {}), + runnerConfigSsmPath: payload.runnerConfigSsmPath, + runnerTokenSsmPath: payload.runnerTokenSsmPath, } satisfies MicrovmRunHookPayloadV1); } @@ -68,9 +87,16 @@ export async function createMicrovmRunners( return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; } let config; + let normalizedGithubRunnerConfig: CreateGitHubRunnerConfig; try { config = { ...loadMicrovmProviderConfig(), ...overrides }; - assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, githubRunnerConfig.ssmTokenPath); + assertMatchingMicrovmRunnerTokenPath(config.runnerTokenSsmPath, githubRunnerConfig.ssmTokenPath); + assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, config.runnerTokenSsmPath); + normalizedGithubRunnerConfig = { + ...githubRunnerConfig, + ssmConfigPath: normalizeMicrovmSsmPath(githubRunnerConfig.ssmConfigPath), + ssmTokenPath: config.runnerTokenSsmPath, + }; } catch (error) { logger.error('Invalid Lambda MicroVM provider configuration', { error }); return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; @@ -82,34 +108,46 @@ export async function createMicrovmRunners( nonRetryableErrorCount: 0, }; const runHookPayload = createMicrovmRunHookPayload({ - runnerConfigSsmPath: githubRunnerConfig.ssmConfigPath, - runnerTokenSsmPath: githubRunnerConfig.ssmTokenPath, + ...(config.imageVersion !== undefined + ? { + imageArn: config.imageIdentifier, + imageVersion: config.imageVersion, + } + : {}), + runnerConfigSsmPath: normalizedGithubRunnerConfig.ssmConfigPath, + runnerTokenSsmPath: normalizedGithubRunnerConfig.ssmTokenPath, }); const environment = process.env.ENVIRONMENT; - const metadataTags = createMicrovmMetadataTags(githubRunnerConfig, environment); + const metadataTags = createMicrovmMetadataTags(normalizedGithubRunnerConfig, environment); for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { let microvmId: string | undefined; try { - microvmId = await runMicrovmRunner({ + const runner = await runMicrovmRunner({ config, environment, runHookPayload, - runnerOwner: githubRunnerConfig.runnerOwner, - runnerType: githubRunnerConfig.runnerType, + runnerOwner: normalizedGithubRunnerConfig.runnerOwner, + runnerType: normalizedGithubRunnerConfig.runnerType, ssmParameterStoreTags: metadataTags, source, }); + microvmId = runner.microvmId; - const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { - getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], - onJitConfigCreated: async (runnerId, metadata) => { - await setMicrovmGithubRunnerMetadata(config.metadataSsmPath, runnerId, metadata); + const failedRunnerIds = await createStartRunnerConfig( + normalizedGithubRunnerConfig, + [microvmId], + githubInstallationClient, + { + getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await setMicrovmGithubRunnerMetadata(config, runnerId, metadata, runner.metadataTags); + }, }, - }); + ); if (failedRunnerIds.includes(microvmId)) { - await terminateMicrovm(microvmId, config.metadataSsmPath).catch((terminationError) => { + await terminateMicrovm(microvmId, config).catch((terminationError) => { logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { error: terminationError, }); @@ -120,7 +158,7 @@ export async function createMicrovmRunners( } } catch (error) { if (microvmId) { - await terminateMicrovm(microvmId, config.metadataSsmPath).catch((terminationError) => { + await terminateMicrovm(microvmId, config).catch((terminationError) => { logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { error: terminationError, }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts index cdae1a58a2..502fc77d05 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -1,14 +1,23 @@ -import { addParameterTags, deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { + assertMatchingMicrovmRunnerTokenPath, assertSeparatedMicrovmMetadataPath, createMicrovmRunnerMetadata, - deleteMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + deleteMicrovmRunnerSsmState, listMicrovmRunnerMetadata, markMicrovmCleanupPending, microvmMetadataParameterName, + microvmRunnerJitParameterName, setMicrovmGithubRunnerMetadata, setMicrovmOrphan, type MicrovmRunnerMetadata, @@ -17,11 +26,19 @@ import { vi.mock('@aws-github-runner/aws-ssm-util', () => ({ addParameterTags: vi.fn(), deleteParameter: vi.fn(), + getParameters: vi.fn(), getParametersByPath: vi.fn(), putParameter: vi.fn(), })); const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const launchTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { return { @@ -48,6 +65,7 @@ beforeEach(() => { vi.useRealTimers(); vi.mocked(deleteParameter).mockResolvedValue(); vi.mocked(addParameterTags).mockResolvedValue(); + vi.mocked(getParameters).mockImplementation(async (names) => new Map([[names[0], '{}']])); vi.mocked(getParametersByPath).mockResolvedValue(new Map()); vi.mocked(putParameter).mockResolvedValue(); }); @@ -56,6 +74,10 @@ describe('MicroVM metadata paths', () => { it('uses one base parameter per validated MicroVM ID', () => { expect(microvmMetadataParameterName(`${metadataSsmPath}/`, 'microvm-123')).toBe(`${metadataSsmPath}/microvm-123`); expect(() => microvmMetadataParameterName(metadataSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + expect(microvmRunnerJitParameterName(`${runnerTokenSsmPath}/`, 'microvm-123')).toBe( + `${runnerTokenSsmPath}/microvm-123`, + ); + expect(() => microvmRunnerJitParameterName(runnerTokenSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); }); it('requires metadata to use a prefix separate from JIT configuration', () => { @@ -66,6 +88,10 @@ describe('MicroVM metadata paths', () => { 'must be separate', ); expect(() => assertSeparatedMicrovmMetadataPath('/runner', '/runner/token')).toThrow('must be separate'); + expect(() => assertMatchingMicrovmRunnerTokenPath(`${runnerTokenSsmPath}/`, runnerTokenSsmPath)).not.toThrow(); + expect(() => assertMatchingMicrovmRunnerTokenPath('/runner/other-token', runnerTokenSsmPath)).toThrow( + 'must match the runner JIT token path', + ); }); }); @@ -74,7 +100,7 @@ describe('MicroVM metadata lifecycle', () => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-08-19T10:00:00.000Z')); - await createMicrovmRunnerMetadata(metadataSsmPath, { + const createdTags = await createMicrovmRunnerMetadata(metadataSsmPath, { microvmId: 'mvm-1', environment: 'unit-test', runnerOwner: 'Codertocat', @@ -118,6 +144,7 @@ describe('MicroVM metadata lifecycle', () => { ], }, ); + expect(createdTags).toEqual(vi.mocked(putParameter).mock.calls[0][3]?.tags); }); it('rejects reserved tag keys and preserves room for late GitHub metadata', async () => { @@ -151,7 +178,26 @@ describe('MicroVM metadata lifecycle', () => { expect(putParameter).not.toHaveBeenCalled(); }); - it('loads active metadata with independent state and cleans expired inactive records', async () => { + it('rejects launch tags whose complete serialized metadata could exceed the Parameter Store value limit', async () => { + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: Array.from({ length: 20 }, (_, index) => ({ + Key: `Custom${index}${'k'.repeat(100)}`, + Value: 'v'.repeat(256), + })), + }), + ).rejects.toThrow('cannot exceed 8192 bytes when serialized'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('loads active metadata and schedules expired or invalid inactive records for two-phase cleanup', async () => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); const active = metadata({ expiresAt: '2026-08-19T12:30:00.000Z' }); @@ -168,50 +214,66 @@ describe('MicroVM metadata lifecycle', () => { ]), ); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ - cleanupMicrovmIds: [], + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-old', 'mvm-invalid'], metadataById: new Map([['mvm-1', { ...active, githubRunnerId: 'github-42', orphan: true }]]), }); expect(getParametersByPath).toHaveBeenCalledWith(metadataSsmPath); - expect(deleteParameter).toHaveBeenCalledTimes(4); - expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-old`); + expect(deleteParameter).not.toHaveBeenCalled(); expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-new`); }); - it('fails closed for invalid metadata or state belonging to an active MicroVM', async () => { + it('fails closed for invalid ownership metadata belonging to an active MicroVM', async () => { vi.mocked(getParametersByPath).mockResolvedValue(new Map([[`${metadataSsmPath}/mvm-1`, '{not-json']])); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( 'invalid ownership metadata', ); + }); + it('schedules provider-owned metadata with invalid orphan state for two-phase cleanup', async () => { vi.mocked(getParametersByPath).mockResolvedValue( new Map([ [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], [`${metadataSsmPath}/mvm-1.orphan`, 'invalid'], ]), ); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( - 'invalid orphan state', - ); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); }); it('propagates metadata path lookup errors so inventory fails closed', async () => { vi.mocked(getParametersByPath).mockRejectedValue(new Error('AccessDenied')); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( - 'AccessDenied', - ); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow('AccessDenied'); }); it('updates GitHub state and adds late GitHub metadata tags to the base parameter', async () => { const runnerLabels = ['self-hosted', 'linux', 'env:unit-test']; - await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-42', - runnerLabels, + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, }); - expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.tags`, expect.any(String), false, { overwrite: true, }); + const tagsValue = vi.mocked(putParameter).mock.calls.find(([name]) => name.endsWith('.tags'))?.[1]; + expect(JSON.parse(tagsValue ?? '{}')).toEqual({ + CostCenter: '1234', + 'ghr:Application': 'github-action-runner', + 'ghr:github_runner_id': 'github-42', + 'ghr:microvm_id': 'mvm-1', + 'ghr:runner_labels': `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }); expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ { Key: 'ghr:github_runner_id', Value: 'github-42' }, { @@ -221,13 +283,53 @@ describe('MicroVM metadata lifecycle', () => { ]); }); + it('revokes JIT configuration when cleanup starts before late metadata is recorded', async () => { + vi.mocked(getParameters).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, '{}'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when ownership metadata is already absent', async () => { + vi.mocked(getParameters).mockResolvedValue(new Map()); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when the post-write ownership fence cannot be read', async () => { + vi.mocked(getParameters).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + it('splits encoded runner labels into SSM-safe tag values', async () => { const runnerLabels = [`label-${'a'.repeat(140)}`, `label-${'b'.repeat(140)}`]; - await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-42', - runnerLabels, - }); + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ { Key: 'ghr:github_runner_id', Value: 'github-42' }, @@ -243,10 +345,15 @@ describe('MicroVM metadata lifecycle', () => { }); it('keeps the GitHub runner ID tag when a runner label is too large', async () => { - await setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-42', - runnerLabels: ['x'.repeat(300)], - }); + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: ['x'.repeat(300)], + }, + launchTags, + ); expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ { Key: 'ghr:github_runner_id', Value: 'github-42' }, @@ -257,16 +364,32 @@ describe('MicroVM metadata lifecycle', () => { vi.mocked(addParameterTags).mockRejectedValue(new Error('AccessDenied')); await expect( - setMicrovmGithubRunnerMetadata(metadataSsmPath, 'mvm-1', { - githubRunnerId: 'github-42', - runnerLabels: [], - }), + setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: [], + }, + launchTags, + ), ).resolves.toBeUndefined(); expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { overwrite: true, }); }); + it('fails JIT setup when the canonical tag-value parameter cannot be written', async () => { + vi.mocked(putParameter).mockImplementation(async (name) => { + if (name.endsWith('.tags')) throw new Error('AccessDenied'); + }); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(addParameterTags).not.toHaveBeenCalled(); + }); + it('updates orphan state without a shared read-modify-write record', async () => { await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { @@ -274,7 +397,7 @@ describe('MicroVM metadata lifecycle', () => { }); }); - it('marks cleanup independently and deletes state before ownership metadata', async () => { + it('marks cleanup independently and deletes JIT plus metadata while retaining the tombstone until last', async () => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); @@ -283,18 +406,28 @@ describe('MicroVM metadata lifecycle', () => { `${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z', false, - { overwrite: true }, ); - await deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1'); + await deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1'); expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, `${metadataSsmPath}/mvm-1.github-runner-id`, `${metadataSsmPath}/mvm-1.orphan`, - `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + `${metadataSsmPath}/mvm-1.tags`, `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, ]); }); + it('does not reset the cleanup grace window when its tombstone already exists', async () => { + vi.mocked(putParameter).mockRejectedValueOnce( + Object.assign(new Error('ParameterAlreadyExists'), { __type: 'ParameterAlreadyExists' }), + ); + + await expect(markMicrovmCleanupPending(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledOnce(); + }); + it('continues deleting metadata when optional parameters are already absent', async () => { vi.mocked(deleteParameter) .mockRejectedValueOnce( @@ -306,12 +439,14 @@ describe('MicroVM metadata lifecycle', () => { ) .mockRejectedValueOnce(Object.assign(new Error('missing parameter'), { name: 'ParameterNotFound' })); - await expect(deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).resolves.toBeUndefined(); expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, `${metadataSsmPath}/mvm-1.github-runner-id`, `${metadataSsmPath}/mvm-1.orphan`, - `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + `${metadataSsmPath}/mvm-1.tags`, `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, ]); }); @@ -323,10 +458,17 @@ describe('MicroVM metadata lifecycle', () => { }); vi.mocked(deleteParameter).mockRejectedValueOnce(error); - await expect(deleteMicrovmRunnerMetadata(metadataSsmPath, 'mvm-1')).rejects.toBe(error); + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).rejects.toBe(error); expect(deleteParameter).toHaveBeenCalledTimes(1); }); + it('deletes only the lane JIT parameter when revoking pending runner configuration', async () => { + await deleteMicrovmRunnerJitConfig(runnerTokenSsmPath, 'mvm-1'); + + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + }); + it('returns tracked and state-only active cleanup requests for termination retry', async () => { vi.mocked(getParametersByPath).mockResolvedValue( new Map([ @@ -340,7 +482,7 @@ describe('MicroVM metadata lifecycle', () => { await expect( listMicrovmRunnerMetadata( - metadataSsmPath, + ssmPaths, states([ ['mvm-1', 'RUNNING'], ['mvm-untracked', 'PENDING'], @@ -367,38 +509,122 @@ describe('MicroVM metadata lifecycle', () => { await expect( listMicrovmRunnerMetadata( - metadataSsmPath, + ssmPaths, states(cleanupIds.map((microvmId): [string, MicrovmState] => [microvmId, 'RUNNING'])), ), ).resolves.toEqual({ cleanupMicrovmIds: cleanupIds, metadataById: new Map() }); }); - it('cleans terminal state-only records and aged markers after inventory no longer sees the MicroVM', async () => { + it('keeps cleanup discoverable through the grace window before deleting JIT and every metadata record', async () => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); vi.mocked(getParametersByPath).mockResolvedValue( new Map([ [`${metadataSsmPath}/mvm-terminal.github-runner-id`, 'github-42'], [`${metadataSsmPath}/mvm-missing.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + [`${metadataSsmPath}/mvm-missing.tags`, '{"ghr:microvm_id":"mvm-missing"}'], [`${metadataSsmPath}/mvm-recent.cleanup-requested-at`, '2026-08-19T11:59:00.000Z'], + [`${metadataSsmPath}/mvm-recent.tags`, '{"ghr:microvm_id":"mvm-recent"}'], ]), ); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual( - { cleanupMicrovmIds: [], metadataById: new Map() }, - ); - expect(deleteParameter).toHaveBeenCalledTimes(8); - expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-terminal`); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-terminal', 'mvm-recent'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-missing`); expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing.tags`); + expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-missing.cleanup-requested-at`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-terminal`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-recent`); expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-recent`); }); + it('deletes invalid ownership metadata after its valid cleanup tombstone ages', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + [`${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, new Map())).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.github-runner-id`, + `${metadataSsmPath}/mvm-invalid.orphan`, + `${metadataSsmPath}/mvm-invalid.tags`, + `${metadataSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, + ]); + }); + + it('repairs an invalid cleanup timestamp before recreating the two-phase cleanup marker', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, 'not-a-timestamp'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.cleanup-requested-at`); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + vi.clearAllMocks(); + vi.setSystemTime(new Date('2026-08-19T12:06:00.000Z')); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + }); + + it('marks a terminal tags-only companion for two-phase cleanup instead of deleting it immediately', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-tags-only.tags`, '{"ghr:microvm_id":"mvm-tags-only"}']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-tags-only', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-tags-only'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + it('fails closed for active state metadata without ownership or a cleanup request', async () => { vi.mocked(getParametersByPath).mockResolvedValue( new Map([[`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42']]), ); - await expect(listMicrovmRunnerMetadata(metadataSsmPath, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( 'state metadata but no ownership metadata', ); }); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts index 4f479d04ff..f447ae135f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -1,5 +1,11 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { addParameterTags, deleteParameter, getParametersByPath, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; import type { CreateGitHubRunnerConfig, GitHubRunnerMetadata, LambdaRunnerSource, RunnerType } from '../../../../core'; @@ -15,14 +21,27 @@ const MAX_RUNNER_LABEL_TAGS = 5; const MAX_BASE_PARAMETER_TAGS = MAX_PARAMETER_TAGS - MAX_RUNNER_LABEL_TAGS - 1; const MAX_TAG_KEY_LENGTH = 128; const MAX_TAG_VALUE_LENGTH = 256; +const MAX_PARAMETER_VALUE_SIZE_IN_BYTES = 8 * 1024; const SSM_TAG_VALUE_PATTERN = /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u; const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; const ORPHAN_SUFFIX = '.orphan'; const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; +const TAGS_SUFFIX = '.tags'; +const METADATA_COMPANION_SUFFIXES = [ + GITHUB_RUNNER_ID_SUFFIX, + ORPHAN_SUFFIX, + CLEANUP_REQUESTED_AT_SUFFIX, + TAGS_SUFFIX, +] as const; const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); type MicrovmMetadataTag = CreateGitHubRunnerConfig['ssmParameterStoreTags'][number]; +export interface MicrovmSsmPaths { + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + export interface MicrovmRunnerMetadata { bypassRemoval?: boolean; createdAt: string; @@ -89,6 +108,32 @@ function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadata return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); } +function serializeParameterTags(tags: MicrovmMetadataTag[]): string { + assertValidParameterTags(tags); + const tagValues: Record = Object.create(null) as Record; + for (const { Key, Value } of [...tags].sort((left, right) => + left.Key < right.Key ? -1 : left.Key > right.Key ? 1 : 0, + )) { + tagValues[Key] = Value; + } + + const value = JSON.stringify(tagValues); + if (Buffer.byteLength(value, 'utf8') > MAX_PARAMETER_VALUE_SIZE_IN_BYTES) { + throw new Error(`MicroVM metadata tags cannot exceed ${MAX_PARAMETER_VALUE_SIZE_IN_BYTES} bytes when serialized`); + } + return value; +} + +function maximumGitHubRunnerMetadataTags(): MicrovmMetadataTag[] { + return [ + { Key: 'ghr:github_runner_id', Value: '0'.repeat(MAX_TAG_VALUE_LENGTH) }, + ...Array.from({ length: MAX_RUNNER_LABEL_TAGS }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: '0'.repeat(MAX_TAG_VALUE_LENGTH), + })), + ]; +} + function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { const configuredTags = mergeParameterTags(input.ssmParameterStoreTags).filter( (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version' && tag.Key !== 'Name', @@ -113,6 +158,7 @@ function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): M `MicroVM metadata cannot have more than ${MAX_BASE_PARAMETER_TAGS} launch tags because ${MAX_RUNNER_LABEL_TAGS + 1} tags are reserved for GitHub runner metadata`, ); } + serializeParameterTags(mergeParameterTags(tags, maximumGitHubRunnerMetadataTags())); return tags; } @@ -168,15 +214,26 @@ function createGitHubRunnerMetadataTags(metadata: GitHubRunnerMetadata): Microvm return tags; } -function normalizedPath(path: string): string { - return path.trim().replace(/\/+$/, ''); +export function normalizeMicrovmSsmPath(path: string): string { + const normalized = path.trim().replace(/\/+$/, ''); + if (!/^\/[A-Za-z0-9_.\-/]+$/.test(normalized) || normalized.includes('//') || normalized.split('/').includes('..')) { + throw new Error(`Invalid SSM parameter path '${path}'`); + } + return normalized; } export function microvmMetadataParameterName(metadataSsmPath: string, microvmId: string): string { if (!MICROVM_ID_PATTERN.test(microvmId)) { throw new Error(`Invalid MicroVM identifier '${microvmId}'`); } - return `${normalizedPath(metadataSsmPath)}/${microvmId}`; + return `${normalizeMicrovmSsmPath(metadataSsmPath)}/${microvmId}`; +} + +export function microvmRunnerJitParameterName(runnerTokenSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(runnerTokenSsmPath)}/${microvmId}`; } function stateParameterName(metadataSsmPath: string, microvmId: string, suffix: string): string { @@ -188,14 +245,15 @@ function metadataParameterNames(metadataSsmPath: string, microvmId: string): str return [ `${baseName}${GITHUB_RUNNER_ID_SUFFIX}`, `${baseName}${ORPHAN_SUFFIX}`, - `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, + `${baseName}${TAGS_SUFFIX}`, baseName, + `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, ]; } export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerTokenSsmPath: string): void { - const metadataPath = normalizedPath(metadataSsmPath); - const runnerTokenPath = normalizedPath(runnerTokenSsmPath); + const metadataPath = normalizeMicrovmSsmPath(metadataSsmPath); + const runnerTokenPath = normalizeMicrovmSsmPath(runnerTokenSsmPath); if ( metadataPath === runnerTokenPath || metadataPath.startsWith(`${runnerTokenPath}/`) || @@ -205,15 +263,25 @@ export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runn } } +export function assertMatchingMicrovmRunnerTokenPath( + configuredRunnerTokenSsmPath: string, + runnerTokenSsmPath: string, +): void { + if (normalizeMicrovmSsmPath(configuredRunnerTokenSsmPath) !== normalizeMicrovmSsmPath(runnerTokenSsmPath)) { + throw new Error('MicroVM provider SSM_TOKEN_PATH must match the runner JIT token path'); + } +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function isParameterError(error: unknown, type: string): boolean { + return error instanceof Error && (error.name === type || ('__type' in error && error.__type === type)); +} + function isParameterNotFound(error: unknown): boolean { - return ( - error instanceof Error && - (error.name === 'ParameterNotFound' || ('__type' in error && error.__type === 'ParameterNotFound')) - ); + return isParameterError(error, 'ParameterNotFound'); } function optionalString(value: unknown): value is string | undefined { @@ -274,7 +342,7 @@ function parseMetadata(value: string, expectedMicrovmId: string): MicrovmRunnerM export async function createMicrovmRunnerMetadata( metadataSsmPath: string, input: CreateMicrovmRunnerMetadataInput, -): Promise { +): Promise { const createdAt = new Date(); const metadata: MicrovmRunnerMetadata = { version: METADATA_VERSION, @@ -291,44 +359,37 @@ export async function createMicrovmRunnerMetadata( ).toISOString(), }; + const metadataTags = createMetadataParameterTags(input); await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false, { - tags: createMetadataParameterTags(input), + tags: metadataTags, }); + return metadataTags; } -function invalidStateReason(parameters: Map, baseName: string): string | undefined { +function invalidOrphanState(parameters: Map, baseName: string): boolean { const orphan = parameters.get(`${baseName}${ORPHAN_SUFFIX}`); - if (orphan !== undefined && orphan !== 'true' && orphan !== 'false') return 'invalid orphan state'; - - const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); - if (cleanupRequestedAt !== undefined && !Number.isFinite(Date.parse(cleanupRequestedAt))) { - return 'invalid cleanup request timestamp'; - } - return undefined; + return orphan !== undefined && orphan !== 'true' && orphan !== 'false'; } -function shouldDeleteMetadata( - metadata: MicrovmRunnerMetadata, - state: MicrovmState | undefined, - cleanupRequestedAt: string | undefined, - now: number, -): boolean { - if (state === 'TERMINATED') return true; - if (state !== undefined) return false; +type CleanupRequestStatus = 'absent' | 'elapsed' | 'invalid' | 'pending'; - const cleanupGraceElapsed = - cleanupRequestedAt !== undefined && Date.parse(cleanupRequestedAt) + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now; - return cleanupGraceElapsed || Date.parse(metadata.expiresAt) <= now; +function cleanupRequestStatus(parameters: Map, baseName: string, now: number): CleanupRequestStatus { + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (cleanupRequestedAt === undefined) return 'absent'; + const requestedAt = Date.parse(cleanupRequestedAt); + if (!Number.isFinite(requestedAt)) return 'invalid'; + return requestedAt + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now ? 'elapsed' : 'pending'; } export async function listMicrovmRunnerMetadata( - metadataSsmPath: string, + paths: MicrovmSsmPaths, microvmStates: ReadonlyMap, ): Promise { + const { metadataSsmPath } = paths; const metadataById = new Map(); const cleanupMicrovmIds = new Set(); - const parameters = await getParametersByPath(normalizedPath(metadataSsmPath)); - const parameterPrefix = `${normalizedPath(metadataSsmPath)}/`; + const parameters = await getParametersByPath(normalizeMicrovmSsmPath(metadataSsmPath)); + const parameterPrefix = `${normalizeMicrovmSsmPath(metadataSsmPath)}/`; const now = Date.now(); const metadataBaseIds = new Set(); const stateParameterIds = new Set(); @@ -336,7 +397,7 @@ export async function listMicrovmRunnerMetadata( for (const parameterName of parameters.keys()) { if (!parameterName.startsWith(parameterPrefix)) continue; - for (const suffix of [GITHUB_RUNNER_ID_SUFFIX, ORPHAN_SUFFIX, CLEANUP_REQUESTED_AT_SUFFIX]) { + for (const suffix of METADATA_COMPANION_SUFFIXES) { if (!parameterName.endsWith(suffix)) continue; const microvmId = parameterName.slice(parameterPrefix.length, -suffix.length); if (MICROVM_ID_PATTERN.test(microvmId)) stateParameterIds.add(microvmId); @@ -351,41 +412,52 @@ export async function listMicrovmRunnerMetadata( metadataBaseIds.add(microvmId); const state = microvmStates.get(microvmId); - const metadata = parseMetadata(value, microvmId); - if (!metadata) { - if (state !== undefined && ACTIVE_STATES.has(state)) { - throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else { + cleanupMicrovmIds.add(microvmId); } - if (state === 'TERMINATED') runnersToDelete.add(microvmId); - else logger.warn(`Ignoring invalid MicroVM runner metadata for '${microvmId}'`); continue; } - const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); - const stateError = invalidStateReason(parameters, baseName); - if (stateError) { + const metadata = parseMetadata(value, microvmId); + if (!metadata) { if (state !== undefined && ACTIVE_STATES.has(state)) { - throw new Error(`Active MicroVM runner '${microvmId}' has ${stateError}`); + throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); } - if (state === 'TERMINATED' || (state === undefined && Date.parse(metadata.expiresAt) <= now)) { - runnersToDelete.add(microvmId); + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); } - logger.warn(`Ignoring MicroVM runner metadata for '${microvmId}' with ${stateError}`); + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling invalid MicroVM runner metadata for '${microvmId}' for cleanup`); continue; } - const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); - if (shouldDeleteMetadata(metadata, state, cleanupRequestedAt, now)) { - runnersToDelete.add(microvmId); + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + logger.warn(`Repairing invalid cleanup request metadata for '${microvmId}'`); continue; } - if (state === undefined || !ACTIVE_STATES.has(state)) continue; + if (invalidOrphanState(parameters, baseName)) { + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling MicroVM runner metadata for '${microvmId}' with invalid orphan state for cleanup`); + continue; + } - if (cleanupRequestedAt !== undefined) { + if (state === 'TERMINATED') { cleanupMicrovmIds.add(microvmId); continue; } + if (state === undefined) { + if (Date.parse(metadata.expiresAt) <= now) cleanupMicrovmIds.add(microvmId); + continue; + } + if (!ACTIVE_STATES.has(state)) continue; metadataById.set(microvmId, { ...metadata, @@ -399,37 +471,37 @@ export async function listMicrovmRunnerMetadata( const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); const state = microvmStates.get(microvmId); - const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); - const stateError = invalidStateReason(parameters, baseName); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); - if (stateError && state !== undefined && ACTIVE_STATES.has(state)) { - throw new Error(`Active MicroVM runner '${microvmId}' has ${stateError}`); + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else if (state === undefined || state === 'TERMINATED' || ACTIVE_STATES.has(state)) { + cleanupMicrovmIds.add(microvmId); + } + continue; } - if (state !== undefined && ACTIVE_STATES.has(state)) { - if (cleanupRequestedAt === undefined) { - throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); + + if (cleanupStatus === 'invalid') { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has an invalid cleanup request timestamp`); } + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); cleanupMicrovmIds.add(microvmId); continue; } - if (state === 'TERMINATED') { - runnersToDelete.add(microvmId); - continue; + + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); } - if (state === undefined) { - const cleanupGraceElapsed = - cleanupRequestedAt !== undefined && - Number.isFinite(Date.parse(cleanupRequestedAt)) && - Date.parse(cleanupRequestedAt) + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now; - if (cleanupRequestedAt === undefined || stateError !== undefined || cleanupGraceElapsed) { - runnersToDelete.add(microvmId); - } + if (state === 'TERMINATED' || state === undefined) { + cleanupMicrovmIds.add(microvmId); } } for (const microvmId of [...runnersToDelete].slice(0, MAX_RECONCILED_RUNNERS)) { try { - await deleteMicrovmRunnerMetadata(metadataSsmPath, microvmId); + await deleteMicrovmRunnerSsmState(paths, microvmId); } catch (error) { logger.warn(`Failed to delete reconciled MicroVM runner metadata '${microvmId}'`, { error }); } @@ -442,24 +514,40 @@ export async function listMicrovmRunnerMetadata( } export async function setMicrovmGithubRunnerMetadata( - metadataSsmPath: string, + paths: MicrovmSsmPaths, microvmId: string, metadata: GitHubRunnerMetadata, + launchTags: MicrovmMetadataTag[], ): Promise { if (!metadata.githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + const baseName = microvmMetadataParameterName(paths.metadataSsmPath, microvmId); + const cleanupMarkerName = `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`; + try { + const parameters = await getParameters([baseName, cleanupMarkerName]); + if (!parameters.has(baseName) || parameters.has(cleanupMarkerName)) { + throw new Error(`MicroVM runner '${microvmId}' is no longer accepting JIT configuration`); + } + } catch (error) { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + throw error; + } + + const githubRunnerTags = createGitHubRunnerMetadataTags(metadata); + const tags = mergeParameterTags(launchTags, githubRunnerTags); + const serializedTags = serializeParameterTags(tags); await putParameter( - stateParameterName(metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), + stateParameterName(paths.metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), metadata.githubRunnerId, false, { overwrite: true, }, ); + await putParameter(stateParameterName(paths.metadataSsmPath, microvmId, TAGS_SUFFIX), serializedTags, false, { + overwrite: true, + }); try { - await addParameterTags( - microvmMetadataParameterName(metadataSsmPath, microvmId), - createGitHubRunnerMetadataTags(metadata), - ); + await addParameterTags(baseName, githubRunnerTags); } catch (error) { logger.error(`Failed to tag MicroVM runner '${microvmId}' with GitHub runner metadata`, { error }); } @@ -472,20 +560,32 @@ export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: strin } export async function markMicrovmCleanupPending(metadataSsmPath: string, microvmId: string): Promise { - await putParameter( - stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), - new Date().toISOString(), - false, - { overwrite: true }, - ); + try { + await putParameter( + stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), + new Date().toISOString(), + false, + ); + } catch (error) { + if (!isParameterError(error, 'ParameterAlreadyExists')) throw error; + } } -export async function deleteMicrovmRunnerMetadata(metadataSsmPath: string, microvmId: string): Promise { - for (const parameterName of metadataParameterNames(metadataSsmPath, microvmId)) { - try { - await deleteParameter(parameterName); - } catch (error) { - if (!isParameterNotFound(error)) throw error; - } +async function deleteParameterIfPresent(parameterName: string): Promise { + try { + await deleteParameter(parameterName); + } catch (error) { + if (!isParameterNotFound(error)) throw error; + } +} + +export async function deleteMicrovmRunnerJitConfig(runnerTokenSsmPath: string, microvmId: string): Promise { + await deleteParameterIfPresent(microvmRunnerJitParameterName(runnerTokenSsmPath, microvmId)); +} + +export async function deleteMicrovmRunnerSsmState(paths: MicrovmSsmPaths, microvmId: string): Promise { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + for (const parameterName of metadataParameterNames(paths.metadataSsmPath, microvmId)) { + await deleteParameterIfPresent(parameterName); } } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts index 02818fb939..f98eb88628 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -15,10 +15,12 @@ vi.mock('./runner-metadata', () => ({ setMicrovmOrphan: vi.fn() })); const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; const providerConfig = { imageIdentifier: imageArn, executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath, + runnerTokenSsmPath, }; beforeEach(() => { @@ -43,7 +45,7 @@ describe('createMicrovmScaleDownProvider', () => { environment: 'unit-test', orphan: undefined, }, - metadataSsmPath, + providerConfig, ); expect(listMicrovmRunners).toHaveBeenNthCalledWith( 2, @@ -51,7 +53,7 @@ describe('createMicrovmScaleDownProvider', () => { environment: 'unit-test', orphan: true, }, - metadataSsmPath, + providerConfig, ); }); @@ -64,7 +66,7 @@ describe('createMicrovmScaleDownProvider', () => { expect(setMicrovmOrphan).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', true); expect(setMicrovmOrphan).toHaveBeenNthCalledWith(2, metadataSsmPath, 'mvm-1', false); - expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', metadataSsmPath); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); }); it('uses the MicroVM boot-time policy', () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts index 9cda68cf53..82038711df 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -5,17 +5,17 @@ import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from '. import { setMicrovmOrphan } from './runner-metadata'; export function createMicrovmScaleDownProvider(): Omit { - const metadataSsmPath = () => loadMicrovmProviderConfig().metadataSsmPath; + const ssmPaths = () => loadMicrovmProviderConfig(); async function list(environment: string, orphan?: boolean): Promise { - return await listMicrovmRunners({ environment, orphan }, metadataSsmPath()); + return await listMicrovmRunners({ environment, orphan }, ssmPaths()); } return { list, bootTimeExceeded: microvmBootTimeExceeded, - markOrphan: async (id) => await setMicrovmOrphan(metadataSsmPath(), id, true), - unmarkOrphan: async (id) => await setMicrovmOrphan(metadataSsmPath(), id, false), - terminate: async (id) => await terminateMicrovm(id, metadataSsmPath()), + markOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, true), + unmarkOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, false), + terminate: async (id) => await terminateMicrovm(id, ssmPaths()), }; } diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts index 58cf080e5e..0111373247 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -10,6 +10,7 @@ declare global { MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; MICROVM_LOG_GROUP: string | undefined; MICROVM_METADATA_SSM_PATH: string; + SSM_TOKEN_PATH: string; } } }