RA-8279: feat(otel): Add OpenTelemetry distributed tracing and ctutils logging integration

.github/workflows/codeql.yml

@@ -58,7 +58,24 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - name: Autobuild
+      env:
+        GOPRIVATE: github.com/digicert/ctutils
+        GONOSUMDB: github.com/digicert/ctutils
+        GONOPROXY: github.com/digicert/ctutils
       uses: github/codeql-action/autobuild@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
 
     # ℹ️ Command-line programs to run using the OS shell.

.github/workflows/golangci-lint.yml

@@ -16,7 +16,24 @@ jobs:
       - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version-file: go.mod
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.CT_APP_ID }}
+          private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+          owner: digicert
+          repositories: ctutils
+
+      - name: Authenticate with private ctutils
+        run: |
+          git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: golangci-lint
+        env: 
+           GOPRIVATE: github.com/digicert/ctutils
+           GONOSUMDB: github.com/digicert/ctutils
+           GONOPROXY: github.com/digicert/ctutils
         uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
         with:
           version: v2.1.6

.github/workflows/govulncheck.yml

@@ -16,8 +16,25 @@ jobs:
     runs-on: ubuntu-latest
     name: Run govulncheck
     steps:
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.CT_APP_ID }}
+          private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+          owner: digicert
+          repositories: ctutils
+
+      - name: Authenticate with private ctutils
+        run: |
+          git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - id: govulncheck
         uses: golang/govulncheck-action@b625fbe08f3bccbe446d94fbf87fcc875a4f50ee # v1.0.4
+        env:
+          GOPRIVATE: github.com/digicert/ctutils
+          GONOSUMDB: github.com/digicert/ctutils
+          GONOPROXY: github.com/digicert/ctutils
         with:
           go-version-file: go.mod
           go-package: ./...

.github/workflows/test_crdb.yaml

@@ -16,6 +16,10 @@ jobs:
       contents: read  # for actions/checkout to fetch code
       pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: github.com/digicert/ctutils
+      GONOSUMDB: github.com/digicert/ctutils
+      GONOPROXY: github.com/digicert/ctutils
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -25,13 +29,30 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
       with:
         version: 'v2.1.6'
         args: ./storage/crdb
 
   unit-test:
     runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: github.com/digicert/ctutils
+      GONOSUMDB: github.com/digicert/ctutils
+      GONOPROXY: github.com/digicert/ctutils
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -41,11 +62,28 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - name: Run tests
       run: go test -v ./storage/crdb/... ./quota/crdbqm/...
 
   integration:
     runs-on: ubuntu-22.04
+    env:
+      GOPRIVATE: github.com/digicert/ctutils
+      GONOSUMDB: github.com/digicert/ctutils
+      GONOPROXY: github.com/digicert/ctutils
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -55,6 +93,19 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - name: Build before tests
       run: go mod download && go build ./...
 

.github/workflows/test_pgdb.yaml

@@ -16,6 +16,10 @@ jobs:
       contents: read  # for actions/checkout to fetch code
       pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: github.com/digicert/ctutils
+      GONOSUMDB: github.com/digicert/ctutils
+      GONOPROXY: github.com/digicert/ctutils
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -25,14 +29,30 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
       with:
         version: 'v2.1.6'
         args: ./storage/postgresql
 
   integration-and-unit-tests:
     runs-on: ubuntu-latest
-
+    env:
+      GOPRIVATE: github.com/digicert/ctutils
+      GONOSUMDB: github.com/digicert/ctutils
+      GONOPROXY: github.com/digicert/ctutils
     services:
       postgres:
         image: postgres
@@ -56,6 +76,19 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Generate token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.CT_APP_ID }}
+        private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+        owner: digicert
+        repositories: ctutils
+
+    - name: Authenticate with private ctutils
+      run: |
+        git config --global url."https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
     - name: Build before tests
       run: go mod download && go build ./...
 

.github/workflows/update-ctutils.yaml

@@ -0,0 +1,95 @@
+name: Update ctutils dependencies
+
+on:
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to install (e.g. v0.1.9-test). Defaults to latest."
+        required: false
+        default: "latest"
+  schedule:
+    - cron: "0 5 * * *" # Runs daily at 05:00 UTC
+
+jobs:
+  update-ctutils:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.CT_APP_ID }}
+          private-key: ${{ secrets.CT_APP_PRIVATE_KEY }}
+          owner: digicert
+          repositories: ctutils
+
+      - name: Update ctutils
+        env:
+          GOPRIVATE: github.com/digicert/ctutils
+          GONOSUMDB: github.com/digicert/ctutils
+          GONOPROXY: github.com/digicert/ctutils
+          GITHUB_APP_TOKEN: ${{ steps.generate_token.outputs.token }}
+          VERSION_INPUT: ${{ github.event.inputs.version || 'latest' }}
+        run: |
+          set -euo pipefail
+
+          # Switch to x-access-token syntax which is standard for App tokens
+          git config --global url."https://x-access-token:$GITHUB_APP_TOKEN@github.com/".insteadOf "https://github.com/"
+
+          RESOLVED_VERSION="$VERSION_INPUT"
+          if [ "$VERSION_INPUT" = "latest" ]; then
+            echo "Resolving latest ctutils release from GitHub API..."
+            # Try getting the official 'latest' release first
+            API_RESPONSE=$(curl -sS -H "Authorization: Bearer $GITHUB_APP_TOKEN" \
+              -H "Accept: application/vnd.github+json" \
+              https://api.github.com/repos/digicert/ctutils/releases/latest || true)
+
+            RESOLVED_VERSION=$(printf '%s' "$API_RESPONSE" | jq -r '.tag_name // empty')
+
+            if [ -z "$RESOLVED_VERSION" ] || [ "$RESOLVED_VERSION" = "null" ]; then
+              echo "Official 'latest' release not found (possibly all pre-releases). Falling back to most recent release..."
+              API_RESPONSE=$(curl -sS -H "Authorization: Bearer $GITHUB_APP_TOKEN" \
+                -H "Accept: application/vnd.github+json" \
+                "https://api.github.com/repos/digicert/ctutils/releases?per_page=1")
+              RESOLVED_VERSION=$(printf '%s' "$API_RESPONSE" | jq -r '.[0].tag_name // empty')
+            fi
+
+            if [ -z "$RESOLVED_VERSION" ] || [ "$RESOLVED_VERSION" = "null" ]; then
+              echo "Error: Unable to resolve any ctutils release tag from GitHub API."
+              echo "API response was: $API_RESPONSE"
+              exit 1
+            fi
+            echo "Resolved version: $RESOLVED_VERSION"
+          fi
+
+          echo "VERSION_USED=$RESOLVED_VERSION" >> "$GITHUB_ENV"
+          echo "Updating ctutils to $RESOLVED_VERSION..."
+          go get github.com/digicert/ctutils@$RESOLVED_VERSION
+          go mod tidy
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore(deps): update ctutils to ${{ env.VERSION_USED }}"
+          committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          branch: chore/update-ctutils-test
+
+          delete-branch: true
+          title: "chore(deps): update ctutils to ${{ env.VERSION_USED }}"
+          body: |
+            Updates [ctutils](https://github.com/digicert/ctutils) to version `${{ env.VERSION_USED }}`.
+
+            Auto-generated by GitHub Actions.
+          labels: |
+            dependencies
+            automated pr

.gitignore

@@ -23,3 +23,6 @@
 /trillian_map_server
 default.etcd
 cockroach-data/
+server/.DS_Store
+.DS_Store
+cmd/trillian_log_server/trillian_log_server

README.md

@@ -18,6 +18,7 @@
  - [Using the Code](#using-the-code)
      - [MySQL Setup](#mysql-setup)
      - [Integration Tests](#integration-tests)
+     - [Observability](#observability)
  - [Working on the Code](#working-on-the-code)
      - [Rebuilding Generated Code](#rebuilding-generated-code)
      - [Updating Dependencies](#updating-dependencies)
@@ -156,6 +157,51 @@ This runs a multi-process test:
 You can find instructions on how to deploy Trillian in [deployment](/deployment)
 and [examples/deployment](/examples/deployment) directories.
 
+### Observability
+
+Trillian log server and signer support OpenTelemetry-compliant distributed
+tracing and structured logging via the [ctutils](https://github.com/digicert/ctutils)
+shared library. This enables end-to-end request tracing and consistent log
+formatting across Trillian backend services.
+
+#### Distributed Tracing
+
+Configuration is via environment variables:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `OTEL_ENABLED` | Enable OpenTelemetry | `false` |
+| `OTEL_EXPORTER` | Exporter type (`otlp` or `stdout`) | `stdout` |
+| `OTEL_COLLECTOR_ENDPOINT` | OTLP collector URL | `localhost:4317` |
+| `OTEL_SERVICE_NAME` | Service name for traces | service-specific |
+| `OTEL_SAMPLE_RATIO` | Sampling ratio (0.0-1.0) | `1.0` |
+
+Values outside the 0.0-1.0 range for `OTEL_SAMPLE_RATIO` are validated and handled by the underlying [`ctutils`](https://github.com/digicert/ctutils) library; consult its documentation for details on how such values are treated.
+
+Example:
+
+```bash
+export OTEL_ENABLED=true
+export OTEL_EXPORTER=otlp
+export OTEL_COLLECTOR_ENDPOINT=http://otel-collector:4317
+export OTEL_SERVICE_NAME=trillian-log-server
+export OTEL_SAMPLE_RATIO=0.1
+```
+
+When enabled, Trillian services will:
+- Accept incoming trace context from gRPC requests
+- Propagate trace context to downstream services
+- Export traces to the configured OTLP collector
+
+#### Logging
+
+The following environment variables control the logging format and level:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `LOG_FORMAT` | Format for log output (`text` or `json`) | `text` |
+| `LOG_LEVEL` | Logging level (`DEBUG`, `INFO`, `WARN`, `ERROR`) | `INFO` |
+
 ## Working on the Code
 
 Developers who want to make changes to the Trillian codebase need some