feat: ci, weekly pipeline report, 2 weeks window to gather drone pipeline logs before expiry

Author: mklos-kw

.github/workflows/weekly_pipeline_report.yml

@@ -0,0 +1,28 @@
+name: Weekly Pipeline Report
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 * * 1" # Every Monday at 8:00 AM UTC
+
+permissions:
+  contents: read
+
+jobs:
+  generate-report:
+    name: Generate Pipeline Report
+    runs-on: ubuntu-latest
+    container: golang:1.25-alpine
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run CI Reporter
+        working-directory: tools/ci-reporter
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          GOWORK=off go run ./cmd/list-failed-commits \
+            --github-token "$GITHUB_TOKEN" \
+            --since 14d \
+            --max-commits 9999999

tools/ci-reporter/cmd/list-failed-commits/main.go

@@ -0,0 +1,239 @@
+// Usage examples:
+//   go run ./cmd/list-failed-commits --max-commits 20
+//   go run ./cmd/list-failed-commits --max-commits 50 --out failed.json
+//   go run ./cmd/list-failed-commits --github-token $GITHUB_TOKEN --max-commits 100
+//   go run ./cmd/list-failed-commits --repo owncloud/ocis --branch master --max-commits 20
+//   go run ./cmd/list-failed-commits --since 7d --max-commits 50
+//   go run ./cmd/list-failed-commits --since 30d --out failed.json
+//   go run ./cmd/list-failed-commits --since 2026-01-01 --max-commits 100
+//   go run ./cmd/list-failed-commits --since 2026-02-15 --out recent.json
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/droneextractor"
+	"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
+	"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/reporter"
+	"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
+)
+
+const (
+	defaultRepo       = "owncloud/ocis"
+	defaultBranch     = "master"
+	defaultMaxCommits = 20
+	requestDelay      = 1 * time.Second
+)
+
+func main() {
+	repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
+	branch := flag.String("branch", defaultBranch, "Branch name")
+	maxCommits := flag.Int("max-commits", defaultMaxCommits, "Maximum commits to check")
+	since := flag.String("since", "", "Date filter: YYYY-MM-DD, RFC3339 timestamp, or 'Nd' for last N days (e.g., '7d', '30d')")
+	outFile := flag.String("out", "", "Output file path (default: stdout)")
+	githubToken := flag.String("github-token", "", "GitHub token (optional; falls back to env GITHUB_TOKEN)")
+	failedPipelineHistory := flag.Bool("failed_pipeline_history", false, "if set, fetch failed_pipeline_history per commit (extra Drone API calls)")
+	flag.Parse()
+
+	token := strings.TrimSpace(*githubToken)
+	if token == "" {
+		token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
+	}
+
+	if token != "" {
+		fmt.Fprintf(os.Stderr, "Using GitHub token\n")
+	} else {
+		fmt.Fprintf(os.Stderr, "No token provided, unauthenticated, 60 req/hr limit\n")
+	}
+
+	command := strings.Join(os.Args[1:], " ")
+	command = redactGitHubToken(command)
+
+	parsedSince, err := util.ParseSinceFlag(*since)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error parsing --since flag: %v\n", err)
+		os.Exit(1)
+	}
+
+	if err := run(*repo, *branch, *maxCommits, parsedSince, *outFile, token, command, *failedPipelineHistory); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(repo, branch string, maxCommits int, since, outFile, token, command string, fetchFailedPipelineHistory bool) error {
+	// Configure Transport to prevent stale connection reuse during long pagination
+	transport := &http.Transport{
+		MaxIdleConns:        10,
+		IdleConnTimeout:     30 * time.Second,
+		DisableKeepAlives:   false,
+		MaxIdleConnsPerHost: 2,
+	}
+	client := &http.Client{
+		Timeout:   60 * time.Second, // Increased for long-running pagination
+		Transport: transport,
+	}
+	githubExtractor := githubextractor.NewExtractor(client, token)
+	droneExtractor := droneextractor.NewExtractor(client)
+
+	fmt.Fprintf(os.Stderr, "Scanning %d commits from %s/%s\n", maxCommits, repo, branch)
+	commits, err := githubExtractor.GetCommits(repo, branch, maxCommits, since)
+	if err != nil {
+		return fmt.Errorf("fetching commits: %w", err)
+	}
+
+	var failedCommits []reporter.FailedCommit
+	for _, commit := range commits {
+		status, err := githubExtractor.GetCommitStatus(repo, commit.SHA)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "⚠  %s: %v\n", commit.SHA[:7], err)
+			time.Sleep(requestDelay)
+			continue
+		}
+
+		if status.State == "failure" || status.State == "error" {
+			var failedCtxs []reporter.StatusContext
+			var pipelineInfo *droneextractor.PipelineInfo
+
+			for _, s := range status.Statuses {
+				if s.State != "success" {
+					// Skip Drone entries - they'll be in pipeline_info instead
+					if droneExtractor.IsDroneURL(s.TargetURL) {
+						info, err := droneExtractor.Extract(s.TargetURL)
+						if err == nil {
+							pipelineInfo = info
+						}
+						// Don't add to failed_contexts to avoid redundancy
+						continue
+					}
+					failedCtxs = append(failedCtxs, toReporterStatusContext(s))
+				}
+			}
+			// TODO: Refactor redundant field assignment - prPipelineHistory always assigned, failedPipelineHistory conditional
+			var failedPipelineHistory []droneextractor.PipelineInfoSummary
+			var prPipelineHistory []droneextractor.PipelineInfoSummary
+			var prNumber int
+			var headRef string
+
+			// Get PR for this commit (master commit -> PR ID -> PR branch)
+			prs, err := githubExtractor.GetPRsForCommit(repo, commit.SHA)
+			if err == nil && len(prs) > 0 {
+				prNumber = prs[0].Number
+				headRef = prs[0].Head.Ref
+
+				// Get failed builds by PR branch (black box: all logic in droneextractor)
+				failedBuilds, err := droneExtractor.GetFailedBuildsByPRBranch(droneextractor.BaseURL, repo, headRef)
+				if err == nil {
+					prPipelineHistory = failedBuilds
+					if fetchFailedPipelineHistory {
+						failedPipelineHistory = failedBuilds
+					}
+				}
+			}
+
+			// Initialize empty slice if flag is set but no PR/builds found
+			if fetchFailedPipelineHistory && failedPipelineHistory == nil {
+				failedPipelineHistory = []droneextractor.PipelineInfoSummary{}
+			}
+
+			subject := extractSubject(commit.Commit.Message)
+			failedCommits = append(failedCommits, reporter.FailedCommit{
+				SHA:                   commit.SHA,
+				Date:                  commit.Commit.Author.Date,
+				Subject:               subject,
+				HTMLURL:               commit.HTMLURL,
+				CombinedState:         status.State,
+				FailedContexts:        failedCtxs,
+				PipelineInfo:          pipelineInfo,
+				FailedPipelineHistory: failedPipelineHistory,
+				PrPipelineHistory:     prPipelineHistory,
+				PR:                    prNumber,
+			})
+
+			// Single line output: ✗ sha subject -> PR #num (branch: name)
+			if prNumber > 0 && headRef != "" {
+				fmt.Fprintf(os.Stderr, "✗ %s  %s -> PR #%d (branch: %s)\n", commit.SHA[:7], subject, prNumber, headRef)
+			} else {
+				fmt.Fprintf(os.Stderr, "✗ %s  %s\n", commit.SHA[:7], subject)
+			}
+		}
+
+		time.Sleep(requestDelay)
+	}
+
+	fmt.Fprintf(os.Stderr, "\nResult: %d failed / %d total\n", len(failedCommits), len(commits))
+
+	var minDate, maxDate string
+	for _, c := range commits {
+		date := c.Commit.Author.Date
+		if minDate == "" || date < minDate {
+			minDate = date
+		}
+		if maxDate == "" || date > maxDate {
+			maxDate = date
+		}
+	}
+
+	metadata := reporter.RaportMetadata{
+		Args:         command,
+		Repo:         repo,
+		Branch:       branch,
+		DateRange:    reporter.DateRange{StartDate: minDate, EndDate: maxDate},
+		TotalCommits: len(commits),
+	}
+
+	report := reporter.GenerateReport(failedCommits, metadata)
+
+	var w io.Writer = os.Stdout
+	if outFile != "" {
+		f, err := os.Create(outFile)
+		if err != nil {
+			return fmt.Errorf("creating output file: %w", err)
+		}
+		defer f.Close()
+		w = f
+		fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", outFile)
+	}
+
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(report); err != nil {
+		return fmt.Errorf("encoding JSON: %w", err)
+	}
+
+	return nil
+}
+
+func extractSubject(message string) string {
+	lines := strings.Split(message, "\n")
+	return strings.TrimSpace(lines[0])
+}
+
+func toReporterStatusContext(ctx githubextractor.StatusContext) reporter.StatusContext {
+	return reporter.StatusContext{
+		Context:     ctx.Context,
+		State:       ctx.State,
+		TargetURL:   ctx.TargetURL,
+		Description: ctx.Description,
+	}
+}
+
+func redactGitHubToken(command string) string {
+	// Match --github-token=<token> or --github-token <token>
+	pattern1 := regexp.MustCompile(`--github-token=[^\s]+`)
+	command = pattern1.ReplaceAllString(command, "--github-token=***")
+
+	pattern2 := regexp.MustCompile(`--github-token\s+[^\s]+`)
+	command = pattern2.ReplaceAllString(command, "--github-token ***")
+
+	return command
+}