diff --git a/.github/workflows/framework-golden-tests-private.yml b/.github/workflows/framework-golden-tests-private.yml new file mode 100644 index 000000000..cc49d1228 --- /dev/null +++ b/.github/workflows/framework-golden-tests-private.yml @@ -0,0 +1,82 @@ +name: Framework Golden Private Tests Examples +# Groups tests that require access to private container registries +on: + push: + +jobs: + test: + defaults: + run: + working-directory: framework/examples/myproject + env: + CTF_JD_IMAGE: "${{secrets.AWS_ACCOUNT_ID_PROD}}.dkr.ecr.us-west-2.amazonaws.com/job-distributor:0.22.1" + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + test: + - name: TestPrivate + config: jd.toml + count: 1 + timeout: 10m + # TODO: sdlc auth +# - name: TestDockerFakes +# config: fake_docker.toml +# count: 1 +# timeout: 10m + name: ${{ matrix.test.name }} + steps: + - name: Checkout repo + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - name: Configure AWS credentials using OIDC + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }} + aws-region: us-west-2 + - name: Login to Amazon ECR + id: login-ecr-private + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + with: + registries: ${{ format('{0},{1}', secrets.AWS_ACCOUNT_ID_SDLC, secrets.AWS_ACCOUNT_ID_PROD) }} + env: + AWS_REGION: us-west-2 + - name: Check for changes in Framework + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + with: + filters: | + src: + - 'framework/**' + - '.github/workflows/framework-golden-tests.yml' + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24.0' + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-modules-${{ hashFiles('framework/examples/myproject/go.sum') }}-${{ runner.os }}-framework-golden-examples + restore-keys: | + go-modules-${{ runner.os }}-framework-golden-examples + go-modules-${{ runner.os }} + - name: Install dependencies + run: go mod download + - name: Run Tests + if: steps.changes.outputs.src == 'true' + env: + CTF_CONFIGS: ${{ matrix.test.config }} + run: | + go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }} + - name: Upload Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: container-logs-${{ matrix.test.name }} + path: framework/examples/myproject/logs + retention-days: 1 diff --git a/.github/workflows/framework-golden-tests.yml b/.github/workflows/framework-golden-tests.yml index b1f2b055c..753549330 100644 --- a/.github/workflows/framework-golden-tests.yml +++ b/.github/workflows/framework-golden-tests.yml @@ -75,7 +75,7 @@ jobs: config: fake.toml count: 1 timeout: 10m - # TODO: sdlc auth + # TODO: sdlc auth (move to framework-golden-tests-private.yml, which has that auth set up) # - name: TestDockerFakes # config: fake_docker.toml # count: 1 diff --git a/framework/.changeset/v0.11.3.md b/framework/.changeset/v0.11.3.md index 2f4c42912..1716c99d0 100644 --- a/framework/.changeset/v0.11.3.md +++ b/framework/.changeset/v0.11.3.md @@ -1 +1 @@ -- Bump the Aptos node image to v1.36.6 \ No newline at end of file +- Bump the Aptos node image to v1.36.6 diff --git a/framework/.changeset/v0.11.5.md b/framework/.changeset/v0.11.5.md new file mode 100644 index 000000000..df1488ea3 --- /dev/null +++ b/framework/.changeset/v0.11.5.md @@ -0,0 +1 @@ +- Enhance JD health checks and add a CI test for it \ No newline at end of file diff --git a/framework/components/jd/grpc_wait_strategy.go b/framework/components/jd/grpc_wait_strategy.go new file mode 100644 index 000000000..b57b43872 --- /dev/null +++ b/framework/components/jd/grpc_wait_strategy.go @@ -0,0 +1,124 @@ +package jd + +import ( + "context" + "fmt" + "time" + + "github.com/docker/go-connections/nat" + tc "github.com/testcontainers/testcontainers-go" + tcwait "github.com/testcontainers/testcontainers-go/wait" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + + "github.com/smartcontractkit/chainlink-testing-framework/framework" +) + +// GRPCHealthStrategy implements a wait strategy for gRPC health checks +type GRPCHealthStrategy struct { + Port nat.Port + PollInterval time.Duration + timeout time.Duration +} + +// NewGRPCHealthStrategy creates a new gRPC health check wait strategy +func NewGRPCHealthStrategy(port nat.Port) *GRPCHealthStrategy { + return &GRPCHealthStrategy{ + Port: port, + PollInterval: 200 * time.Millisecond, + timeout: 3 * time.Minute, + } +} + +// WithTimeout sets the timeout for the gRPC health check strategy +func (g *GRPCHealthStrategy) WithTimeout(timeout time.Duration) *GRPCHealthStrategy { + g.timeout = timeout + return g +} + +// WithPollInterval sets the poll interval for the gRPC health check strategy +func (g *GRPCHealthStrategy) WithPollInterval(interval time.Duration) *GRPCHealthStrategy { + g.PollInterval = interval + return g +} + +// WaitUntilReady implements Strategy.WaitUntilReady +func (g *GRPCHealthStrategy) WaitUntilReady(ctx context.Context, target tcwait.StrategyTarget) error { + ctx, cancel := context.WithTimeout(ctx, g.timeout) + defer cancel() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(g.PollInterval): + // Check if container is still running + state, err := target.State(ctx) + if err != nil { + return err + } + if !state.Running { + return fmt.Errorf("container is not running: %s", state.Status) + } + + // Get host and port + host, err := framework.GetHost(target.(tc.Container)) //nolint:contextcheck //don't want modify the signature of GetHost() yet + if err != nil { + continue + } + + mappedPort, err := target.MappedPort(ctx, g.Port) + if err != nil { + continue + } + + // Attempt gRPC health check + address := fmt.Sprintf("%s:%s", host, mappedPort.Port()) + if err := g.checkHealth(ctx, address); err == nil { + return nil + } + } + } +} + +// checkHealth performs the actual gRPC health check +func (g *GRPCHealthStrategy) checkHealth(ctx context.Context, address string) error { + // Create a short timeout for the individual check + checkCtx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + // Use plaintext/insecure connection (standard for local testing and health checks) + return g.tryHealthCheck(checkCtx, address, insecure.NewCredentials()) +} + +// tryHealthCheck attempts a health check with specific credentials +func (g *GRPCHealthStrategy) tryHealthCheck(ctx context.Context, address string, creds credentials.TransportCredentials) error { + // Build dial options similar to the working JD connection code + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(creds), + } + + // Create the gRPC client connection + conn, err := grpc.NewClient(address, opts...) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + + // Create health check client + healthClient := grpc_health_v1.NewHealthClient(conn) + + // Perform health check + resp, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("service not serving, status: %v", resp.Status) + } + + return nil +} diff --git a/framework/components/jd/jd.go b/framework/components/jd/jd.go index 65496ad83..554f6de37 100644 --- a/framework/components/jd/jd.go +++ b/framework/components/jd/jd.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "os" + "time" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" tcwait "github.com/testcontainers/testcontainers-go/wait" "github.com/smartcontractkit/chainlink-testing-framework/framework" @@ -19,6 +21,7 @@ const ( GRPCPort string = "14231" CSAEncryptionKey string = "!PASsword000!" WSRPCPort string = "8080" + WSRPCHealthPort string = "8081" ) type Input struct { @@ -75,6 +78,9 @@ func NewJD(in *Input) (*Output, error) { if jdImg != "" { in.Image = jdImg } + if in.WSRPCPort == WSRPCHealthPort { + return nil, fmt.Errorf("wsrpc port cannot be the same as wsrpc health port") + } if in.DBInput == nil { in.DBInput = defaultJDDB() } @@ -84,7 +90,8 @@ func NewJD(in *Input) (*Output, error) { return nil, err } containerName := framework.DefaultTCName("jd") - bindPort := fmt.Sprintf("%s/tcp", in.GRPCPort) + grpcPort := fmt.Sprintf("%s/tcp", in.GRPCPort) + wsHealthPort := fmt.Sprintf("%s/tcp", WSRPCHealthPort) req := tc.ContainerRequest{ Name: containerName, Image: in.Image, @@ -93,11 +100,11 @@ func NewJD(in *Input) (*Output, error) { NetworkAliases: map[string][]string{ framework.DefaultNetworkName: {containerName}, }, - ExposedPorts: []string{bindPort}, + ExposedPorts: []string{grpcPort, wsHealthPort}, HostConfigModifier: func(h *container.HostConfig) { // JobDistributor service is isolated from internet by default! framework.NoDNS(true, h) - h.PortBindings = framework.MapTheSamePort(bindPort) + h.PortBindings = framework.MapTheSamePort(grpcPort) }, Env: map[string]string{ "DATABASE_URL": pgOut.JDInternalURL, @@ -107,6 +114,13 @@ func NewJD(in *Input) (*Output, error) { }, WaitingFor: tcwait.ForAll( tcwait.ForListeningPort(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))), + wait.ForHTTP("/healthz"). + WithPort(nat.Port(fmt.Sprintf("%s/tcp", WSRPCHealthPort))). // WSRPC health endpoint uses different port than WSRPC + WithStartupTimeout(1*time.Minute). + WithPollInterval(200*time.Millisecond), + NewGRPCHealthStrategy(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))). + WithTimeout(1*time.Minute). + WithPollInterval(200*time.Millisecond), ), } if req.Image == "" { diff --git a/framework/components/jd/jd_test.go b/framework/examples/myproject/jd_test.go similarity index 51% rename from framework/components/jd/jd_test.go rename to framework/examples/myproject/jd_test.go index 59eda4a8b..e411bcdc4 100644 --- a/framework/components/jd/jd_test.go +++ b/framework/examples/myproject/jd_test.go @@ -1,24 +1,20 @@ -package jd_test +package examples import ( "os" "testing" - "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-testing-framework/framework" "github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd" + "github.com/stretchr/testify/require" ) -// here we only test that we can boot up JD -// client examples are under "examples" dir -// since JD is private this env var should be set locally and in CI -// TODO: add ComponentDocker prefix to turn this on when we'll have access to ECRs -func TestJD(t *testing.T) { +func TestPrivateJd(t *testing.T) { err := framework.DefaultNetwork(nil) require.NoError(t, err) _, err = jd.NewJD(&jd.Input{ - Image: os.Getenv("CTF_JD_IMAGE"), + Image: os.Getenv("CTF_JD_IMAGE"), + CSAEncryptionKey: "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44", // random key for tests }) require.NoError(t, err) } diff --git a/framework/examples/myproject_cll/go.mod b/framework/examples/myproject_cll/go.mod index b76ca27c1..af0e45aa3 100644 --- a/framework/examples/myproject_cll/go.mod +++ b/framework/examples/myproject_cll/go.mod @@ -119,6 +119,7 @@ require ( golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/grpc v1.72.2 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/guregu/null.v4 v4.0.0 // indirect diff --git a/framework/examples/myproject_cll/go.sum b/framework/examples/myproject_cll/go.sum index 1a47b4694..bbd5b33a4 100644 --- a/framework/examples/myproject_cll/go.sum +++ b/framework/examples/myproject_cll/go.sum @@ -334,6 +334,8 @@ go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/Wgbsd go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= diff --git a/framework/go.mod b/framework/go.mod index 8dffbe3c7..ea73f2a15 100644 --- a/framework/go.mod +++ b/framework/go.mod @@ -31,6 +31,7 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 go.uber.org/multierr v1.11.0 golang.org/x/sync v0.13.0 + google.golang.org/grpc v1.71.0 gopkg.in/guregu/null.v4 v4.0.0 ) @@ -123,7 +124,6 @@ require ( golang.org/x/text v0.24.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect - google.golang.org/grpc v1.71.0 // indirect google.golang.org/protobuf v1.36.4 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/framework/go.sum b/framework/go.sum index 2da3510d2..c0aac62c3 100644 --- a/framework/go.sum +++ b/framework/go.sum @@ -138,6 +138,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -339,6 +341,8 @@ go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/ go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=