better JD health-checks

.github/workflows/framework-golden-tests-private.yml

@@ -0,0 +1,82 @@
+name: Framework Golden Private Tests Examples
+# Groups tests that require access to private container registries
+on:
+  push:
+
+jobs:
+  test:
+    defaults:
+      run:
+        working-directory: framework/examples/myproject
+    env:
+      CTF_JD_IMAGE: "${{secrets.AWS_ACCOUNT_ID_PROD}}.dkr.ecr.us-west-2.amazonaws.com/job-distributor:0.22.1"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - name: TestPrivate
+            config: jd.toml
+            count: 1
+            timeout: 10m
+            # TODO: sdlc auth
+#          - name: TestDockerFakes
+#            config: fake_docker.toml
+#            count: 1
+#            timeout: 10m
+    name: ${{ matrix.test.name }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
+          aws-region: us-west-2
+      - name: Login to Amazon ECR
+        id: login-ecr-private
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+        with:
+          registries: ${{ format('{0},{1}', secrets.AWS_ACCOUNT_ID_SDLC, secrets.AWS_ACCOUNT_ID_PROD) }}
+        env:
+          AWS_REGION: us-west-2
+      - name: Check for changes in Framework
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'framework/**'
+              - '.github/workflows/framework-golden-tests.yml'
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-modules-${{ hashFiles('framework/examples/myproject/go.sum') }}-${{ runner.os }}-framework-golden-examples
+          restore-keys: |
+            go-modules-${{ runner.os }}-framework-golden-examples
+            go-modules-${{ runner.os }}
+      - name: Install dependencies
+        run: go mod download
+      - name: Run Tests
+        if: steps.changes.outputs.src == 'true'
+        env:
+          CTF_CONFIGS: ${{ matrix.test.config }}
+        run: |
+          go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }}
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: container-logs-${{ matrix.test.name }}
+          path: framework/examples/myproject/logs
+          retention-days: 1

.github/workflows/framework-golden-tests.yml

@@ -75,7 +75,7 @@ jobs:
             config: fake.toml
             count: 1
             timeout: 10m
-            # TODO: sdlc auth
+            # TODO: sdlc auth (move to framework-golden-tests-private.yml, which has that auth set up)
 #          - name: TestDockerFakes
 #            config: fake_docker.toml
 #            count: 1

framework/.changeset/v0.11.3.md

@@ -1 +1 @@
-- Bump the Aptos node image to v1.36.6
+- Bump the Aptos node image to v1.36.6

framework/.changeset/v0.11.5.md

@@ -0,0 +1 @@
+- Enhance JD health checks and add a CI test for it

framework/components/jd/grpc_wait_strategy.go

@@ -0,0 +1,124 @@
+package jd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/docker/go-connections/nat"
+	tc "github.com/testcontainers/testcontainers-go"
+	tcwait "github.com/testcontainers/testcontainers-go/wait"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/health/grpc_health_v1"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+)
+
+// GRPCHealthStrategy implements a wait strategy for gRPC health checks
+type GRPCHealthStrategy struct {
+	Port         nat.Port
+	PollInterval time.Duration
+	timeout      time.Duration
+}
+
+// NewGRPCHealthStrategy creates a new gRPC health check wait strategy
+func NewGRPCHealthStrategy(port nat.Port) *GRPCHealthStrategy {
+	return &GRPCHealthStrategy{
+		Port:         port,
+		PollInterval: 200 * time.Millisecond,
+		timeout:      3 * time.Minute,
+	}
+}
+
+// WithTimeout sets the timeout for the gRPC health check strategy
+func (g *GRPCHealthStrategy) WithTimeout(timeout time.Duration) *GRPCHealthStrategy {
+	g.timeout = timeout
+	return g
+}
+
+// WithPollInterval sets the poll interval for the gRPC health check strategy
+func (g *GRPCHealthStrategy) WithPollInterval(interval time.Duration) *GRPCHealthStrategy {
+	g.PollInterval = interval
+	return g
+}
+
+// WaitUntilReady implements Strategy.WaitUntilReady
+func (g *GRPCHealthStrategy) WaitUntilReady(ctx context.Context, target tcwait.StrategyTarget) error {
+	ctx, cancel := context.WithTimeout(ctx, g.timeout)
+	defer cancel()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(g.PollInterval):
+			// Check if container is still running
+			state, err := target.State(ctx)
+			if err != nil {
+				return err
+			}
+			if !state.Running {
+				return fmt.Errorf("container is not running: %s", state.Status)
+			}
+
+			// Get host and port
+			host, err := framework.GetHost(target.(tc.Container)) //nolint:contextcheck //don't want modify the signature of GetHost() yet
+			if err != nil {
+				continue
+			}
+
+			mappedPort, err := target.MappedPort(ctx, g.Port)
+			if err != nil {
+				continue
+			}
+
+			// Attempt gRPC health check
+			address := fmt.Sprintf("%s:%s", host, mappedPort.Port())
+			if err := g.checkHealth(ctx, address); err == nil {
+				return nil
+			}
+		}
+	}
+}
+
+// checkHealth performs the actual gRPC health check
+func (g *GRPCHealthStrategy) checkHealth(ctx context.Context, address string) error {
+	// Create a short timeout for the individual check
+	checkCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
+	defer cancel()
+
+	// Use plaintext/insecure connection (standard for local testing and health checks)
+	return g.tryHealthCheck(checkCtx, address, insecure.NewCredentials())
+}
+
+// tryHealthCheck attempts a health check with specific credentials
+func (g *GRPCHealthStrategy) tryHealthCheck(ctx context.Context, address string, creds credentials.TransportCredentials) error {
+	// Build dial options similar to the working JD connection code
+	opts := []grpc.DialOption{
+		grpc.WithTransportCredentials(creds),
+	}
+
+	// Create the gRPC client connection
+	conn, err := grpc.NewClient(address, opts...)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = conn.Close() }()
+
+	// Create health check client
+	healthClient := grpc_health_v1.NewHealthClient(conn)
+
+	// Perform health check
+	resp, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
+	if err != nil {
+		return err
+	}
+
+	if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING {
+		return fmt.Errorf("service not serving, status: %v", resp.Status)
+	}
+
+	return nil
+}

framework/components/jd/jd.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
 	tc "github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
 	tcwait "github.com/testcontainers/testcontainers-go/wait"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
@@ -19,6 +21,7 @@ const (
 	GRPCPort         string = "14231"
 	CSAEncryptionKey string = "!PASsword000!"
 	WSRPCPort        string = "8080"
+	WSRPCHealthPort  string = "8081"
 )
 
 type Input struct {
@@ -75,6 +78,9 @@ func NewJD(in *Input) (*Output, error) {
 	if jdImg != "" {
 		in.Image = jdImg
 	}
+	if in.WSRPCPort == WSRPCHealthPort {
+		return nil, fmt.Errorf("wsrpc port cannot be the same as wsrpc health port")
+	}
 	if in.DBInput == nil {
 		in.DBInput = defaultJDDB()
 	}
@@ -84,7 +90,8 @@ func NewJD(in *Input) (*Output, error) {
 		return nil, err
 	}
 	containerName := framework.DefaultTCName("jd")
-	bindPort := fmt.Sprintf("%s/tcp", in.GRPCPort)
+	grpcPort := fmt.Sprintf("%s/tcp", in.GRPCPort)
+	wsHealthPort := fmt.Sprintf("%s/tcp", WSRPCHealthPort)
 	req := tc.ContainerRequest{
 		Name:     containerName,
 		Image:    in.Image,
@@ -93,11 +100,11 @@ func NewJD(in *Input) (*Output, error) {
 		NetworkAliases: map[string][]string{
 			framework.DefaultNetworkName: {containerName},
 		},
-		ExposedPorts: []string{bindPort},
+		ExposedPorts: []string{grpcPort, wsHealthPort},
 		HostConfigModifier: func(h *container.HostConfig) {
 			// JobDistributor service is isolated from internet by default!
 			framework.NoDNS(true, h)
-			h.PortBindings = framework.MapTheSamePort(bindPort)
+			h.PortBindings = framework.MapTheSamePort(grpcPort)
 		},
 		Env: map[string]string{
 			"DATABASE_URL":              pgOut.JDInternalURL,
@@ -107,6 +114,13 @@ func NewJD(in *Input) (*Output, error) {
 		},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForListeningPort(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))),
+			wait.ForHTTP("/healthz").
+				WithPort(nat.Port(fmt.Sprintf("%s/tcp", WSRPCHealthPort))). // WSRPC health endpoint uses different port than WSRPC
+				WithStartupTimeout(1*time.Minute).
+				WithPollInterval(200*time.Millisecond),
+			NewGRPCHealthStrategy(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))).
+				WithTimeout(1*time.Minute).
+				WithPollInterval(200*time.Millisecond),
 		),
 	}
 	if req.Image == "" {

framework/components/jd/jd_test.go → framework/examples/myproject/jd_test.go

@@ -1,24 +1,20 @@
-package jd_test
+package examples
 
 import (
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd"
+	"github.com/stretchr/testify/require"
 )
 
-// here we only test that we can boot up JD
-// client examples are under "examples" dir
-// since JD is private this env var should be set locally and in CI
-// TODO: add ComponentDocker prefix to turn this on when we'll have access to ECRs
-func TestJD(t *testing.T) {
+func TestPrivateJd(t *testing.T) {
 	err := framework.DefaultNetwork(nil)
 	require.NoError(t, err)
 	_, err = jd.NewJD(&jd.Input{
-		Image: os.Getenv("CTF_JD_IMAGE"),
+		Image:            os.Getenv("CTF_JD_IMAGE"),
+		CSAEncryptionKey: "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44", // random key for tests
 	})
 	require.NoError(t, err)
 }

framework/examples/myproject_cll/go.mod

@@ -119,6 +119,7 @@ require (
 	golang.org/x/text v0.26.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	google.golang.org/grpc v1.72.2 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/guregu/null.v4 v4.0.0 // indirect

framework/examples/myproject_cll/go.sum

@@ -334,6 +334,8 @@ go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/Wgbsd
 go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
 go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
 go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
+go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
 go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
 go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
 go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=

framework/go.mod

@@ -31,6 +31,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.35.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/sync v0.13.0
+	google.golang.org/grpc v1.71.0
 	gopkg.in/guregu/null.v4 v4.0.0
 )
 
@@ -123,7 +124,6 @@ require (
 	golang.org/x/text v0.24.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
-	google.golang.org/grpc v1.71.0 // indirect
 	google.golang.org/protobuf v1.36.4 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

framework/go.sum

@@ -138,6 +138,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -339,6 +341,8 @@ go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/
 go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
 go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
 go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
+go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
+go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
 go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
 go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=