Introduce extension API to ShellSentry (#40)

Author: ZacSweers

api/kotlin-cli-util.api

@@ -37,6 +37,25 @@ public final class slack/cli/Toml {
 	public final fun parseVersion (Ljava/io/File;)Ljava/util/Map;
 }
 
+public final class slack/cli/shellsentry/AnalysisResult {
+	public fun <init> (Ljava/lang/String;Ljava/lang/String;Lslack/cli/shellsentry/RetrySignal;ILkotlin/jvm/functions/Function1;)V
+	public final fun component1 ()Ljava/lang/String;
+	public final fun component2 ()Ljava/lang/String;
+	public final fun component3 ()Lslack/cli/shellsentry/RetrySignal;
+	public final fun component4 ()I
+	public final fun component5 ()Lkotlin/jvm/functions/Function1;
+	public final fun copy (Ljava/lang/String;Ljava/lang/String;Lslack/cli/shellsentry/RetrySignal;ILkotlin/jvm/functions/Function1;)Lslack/cli/shellsentry/AnalysisResult;
+	public static synthetic fun copy$default (Lslack/cli/shellsentry/AnalysisResult;Ljava/lang/String;Ljava/lang/String;Lslack/cli/shellsentry/RetrySignal;ILkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lslack/cli/shellsentry/AnalysisResult;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getConfidence ()I
+	public final fun getExplanation ()Ljava/lang/String;
+	public final fun getMessage ()Ljava/lang/String;
+	public final fun getRetrySignal ()Lslack/cli/shellsentry/RetrySignal;
+	public final fun getThrowableMaker ()Lkotlin/jvm/functions/Function1;
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class slack/cli/shellsentry/Issue {
 	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lslack/cli/shellsentry/RetrySignal;)V
 	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lslack/cli/shellsentry/RetrySignal;Ljava/lang/String;)V
@@ -71,6 +90,11 @@ public final class slack/cli/shellsentry/IssueJsonAdapter : com/squareup/moshi/J
 	public fun toString ()Ljava/lang/String;
 }
 
+public abstract class slack/cli/shellsentry/NoStacktraceThrowable : java/lang/Throwable {
+	public fun <init> (Ljava/lang/String;)V
+	public fun fillInStackTrace ()Ljava/lang/Throwable;
+}
+
 public abstract interface class slack/cli/shellsentry/RetrySignal {
 }
 
@@ -122,10 +146,10 @@ public final class slack/cli/shellsentry/RetrySignal_RetryDelayedJsonAdapter : c
 
 public final class slack/cli/shellsentry/ShellSentry {
 	public static final field Companion Lslack/cli/shellsentry/ShellSentry$Companion;
-	public fun <init> (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;)V
-	public synthetic fun <init> (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun copy (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;)Lslack/cli/shellsentry/ShellSentry;
-	public static synthetic fun copy$default (Lslack/cli/shellsentry/ShellSentry;Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lslack/cli/shellsentry/ShellSentry;
+	public fun <init> (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;Ljava/util/List;)V
+	public synthetic fun <init> (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun copy (Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;Ljava/util/List;)Lslack/cli/shellsentry/ShellSentry;
+	public static synthetic fun copy$default (Lslack/cli/shellsentry/ShellSentry;Ljava/lang/String;Ljava/nio/file/Path;Ljava/nio/file/Path;Ljava/lang/String;Lslack/cli/shellsentry/ShellSentryConfig;ZZZLkotlin/jvm/functions/Function1;Ljava/util/List;ILjava/lang/Object;)Lslack/cli/shellsentry/ShellSentry;
 	public fun equals (Ljava/lang/Object;)Z
 	public final fun exec ()V
 	public fun hashCode ()I
@@ -143,16 +167,18 @@ public final class slack/cli/shellsentry/ShellSentryCli : com/github/ajalt/clikt
 
 public final class slack/cli/shellsentry/ShellSentryConfig {
 	public fun <init> ()V
-	public fun <init> (ILjava/lang/String;Ljava/util/List;)V
-	public synthetic fun <init> (ILjava/lang/String;Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun <init> (ILjava/lang/String;Ljava/util/List;I)V
+	public synthetic fun <init> (ILjava/lang/String;Ljava/util/List;IILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public final fun component1 ()I
 	public final fun component2 ()Ljava/lang/String;
 	public final fun component3 ()Ljava/util/List;
-	public final fun copy (ILjava/lang/String;Ljava/util/List;)Lslack/cli/shellsentry/ShellSentryConfig;
-	public static synthetic fun copy$default (Lslack/cli/shellsentry/ShellSentryConfig;ILjava/lang/String;Ljava/util/List;ILjava/lang/Object;)Lslack/cli/shellsentry/ShellSentryConfig;
+	public final fun component4 ()I
+	public final fun copy (ILjava/lang/String;Ljava/util/List;I)Lslack/cli/shellsentry/ShellSentryConfig;
+	public static synthetic fun copy$default (Lslack/cli/shellsentry/ShellSentryConfig;ILjava/lang/String;Ljava/util/List;IILjava/lang/Object;)Lslack/cli/shellsentry/ShellSentryConfig;
 	public fun equals (Ljava/lang/Object;)Z
 	public final fun getGradleEnterpriseServer ()Ljava/lang/String;
 	public final fun getKnownIssues ()Ljava/util/List;
+	public final fun getMinConfidence ()I
 	public final fun getVersion ()I
 	public fun hashCode ()I
 	public fun toString ()Ljava/lang/String;
@@ -165,3 +191,7 @@ public final class slack/cli/shellsentry/ShellSentryConfigJsonAdapter : com/squa
 	public fun toString ()Ljava/lang/String;
 }
 
+public abstract interface class slack/cli/shellsentry/ShellSentryExtension {
+	public abstract fun check (Ljava/lang/String;IZLjava/nio/file/Path;)Lslack/cli/shellsentry/AnalysisResult;
+}
+

src/main/kotlin/slack/cli/shellsentry/Issue.kt

@@ -84,9 +84,12 @@ constructor(
  * Base class for an issue that can be reported to Bugsnag. This is a [Throwable] for BugSnag
  * purposes but doesn't fill in a stacktrace.
  */
-internal class IssueThrowable(issue: Issue) : Throwable(issue.message) {
+public abstract class NoStacktraceThrowable(message: String) : Throwable(message) {
   override fun fillInStackTrace(): Throwable {
     // Do nothing, the stacktrace isn't relevant for these!
     return this
   }
 }
+
+/** Common [Throwable] for all [Issue]s. This is used for reporting to Bugsnag. */
+internal class KnownIssue(issue: Issue) : NoStacktraceThrowable(issue.message)

src/main/kotlin/slack/cli/shellsentry/ResultProcessor.kt

@@ -46,9 +46,11 @@ internal class ResultProcessor(
   private val bugsnagKey: String?,
   private val config: ShellSentryConfig,
   private val echo: (String) -> Unit,
+  private val extensions: List<ShellSentryExtension> = emptyList(),
 ) {
 
-  fun process(logFile: Path, isAfterRetry: Boolean): RetrySignal {
+  @Suppress("LongMethod", "CyclomaticComplexMethod", "NestedBlockDepth", "ReturnCount")
+  fun process(command: String, exitCode: Int, logFile: Path, isAfterRetry: Boolean): RetrySignal {
     echo("Processing CI log from ${logFile.absolutePathString()}")
 
     val bugsnag: Bugsnag? by lazy { bugsnagKey?.let { key -> createBugsnag(key) } }
@@ -62,7 +64,7 @@ internal class ResultProcessor(
         // Report to bugsnag. Shared common Throwable but with different messages.
         bugsnag?.let {
           verboseEcho("Reporting to bugsnag: $retrySignal")
-          it.notify(IssueThrowable(issue), Severity.ERROR) { report ->
+          it.notify(KnownIssue(issue), Severity.ERROR) { report ->
             // Group by the throwable message
             report.setGroupingHash(issue.groupingHash)
             report.addToTab("Run Info", "After-Retry", isAfterRetry)
@@ -82,7 +84,42 @@ internal class ResultProcessor(
       }
     }
 
-    // TODO some day log these into bugsnag too?
+    echo("No matching issues from config.json")
+    if (extensions.isNotEmpty()) {
+      echo("Checking extensions")
+      for (extension in extensions) {
+        val result = extension.check(command, exitCode, isAfterRetry, logFile) ?: continue
+
+        verboseEcho(result.message)
+        verboseEcho(result.explanation)
+
+        if (
+          result.retrySignal != RetrySignal.Unknown && result.confidence >= config.minConfidence
+        ) {
+          // Report to bugsnag. Shared common Throwable but with different messages.
+          bugsnag?.let {
+            verboseEcho("Reporting to bugsnag: ${result.retrySignal}")
+            it.notify(result.throwableMaker(result.message), Severity.ERROR) { report ->
+              report.addToTab("Run Info", "After-Retry", isAfterRetry)
+              config.gradleEnterpriseServer?.let(logLinesReversed::parseBuildScan)?.let { scanLink
+                ->
+                report.addToTab("Run Info", "Build-Scan", scanLink)
+              }
+              report.addToTab("Extensions", "Explanation", result.explanation)
+            }
+          }
+            ?: run { verboseEcho("Skipping bugsnag reporting: ${result.retrySignal}") }
+
+          if (result.retrySignal is RetrySignal.Ack) {
+            echo("Acknowledging issue but cannot retry: ${result.message}")
+          } else {
+            echo("Found retry signal: ${result.retrySignal}")
+          }
+          return result.retrySignal
+        }
+      }
+    }
+
     echo("No actionable items found in ${logFile.name}")
     return RetrySignal.Unknown
   }

src/main/kotlin/slack/cli/shellsentry/ShellSentry.kt

@@ -53,6 +53,7 @@ public data class ShellSentry(
   private val debug: Boolean = false,
   private val noExit: Boolean = false,
   private val logger: (String) -> Unit = ::println,
+  private val extensions: List<ShellSentryExtension> = emptyList()
 ) {
 
   @Suppress("CyclomaticComplexMethod", "LongMethod")
@@ -70,9 +71,9 @@ public data class ShellSentry(
       )
 
       logger("Processing CI failure")
-      val resultProcessor = ResultProcessor(verbose, bugsnagKey, config, logger)
+      val resultProcessor = ResultProcessor(verbose, bugsnagKey, config, logger, extensions)
 
-      when (val retrySignal = resultProcessor.process(logFile, false)) {
+      when (val retrySignal = resultProcessor.process(command, exitCode, logFile, false)) {
         is RetrySignal.Ack,
         RetrySignal.Unknown -> {
           logger("Processor exited with 0, exiting with original exit code...")
@@ -89,7 +90,12 @@ public data class ShellSentry(
           logFile = secondResult.outputFile
           if (secondResult.exitCode != 0) {
             // Process the second failure, then bounce out
-            resultProcessor.process(secondResult.outputFile, isAfterRetry = true)
+            resultProcessor.process(
+              command,
+              secondResult.exitCode,
+              secondResult.outputFile,
+              isAfterRetry = true
+            )
           }
         }
         is RetrySignal.RetryImmediately -> {
@@ -100,7 +106,12 @@ public data class ShellSentry(
           logFile = secondResult.outputFile
           if (secondResult.exitCode != 0) {
             // Process the second failure, then bounce out
-            resultProcessor.process(secondResult.outputFile, isAfterRetry = true)
+            resultProcessor.process(
+              command,
+              secondResult.exitCode,
+              secondResult.outputFile,
+              isAfterRetry = true
+            )
           }
         }
       }

src/main/kotlin/slack/cli/shellsentry/ShellSentryConfig.kt

@@ -29,6 +29,11 @@ public data class ShellSentryConfig(
   @Json(name = "known_issues")
   val knownIssues: List<Issue> =
     KnownIssues::class.declaredMemberProperties.map { it.get(KnownIssues) as Issue },
+  /**
+   * A minimum confidence level on a scale of [0-100] to accept. [AnalysisResult]s from
+   * [ShellSentryExtension]s with lower confidence than this will be discarded.
+   */
+  @Json(name = "min_confidence") val minConfidence: Int = 75
 ) {
   init {
     check(version == CURRENT_VERSION) {

src/main/kotlin/slack/cli/shellsentry/ShellSentryExtension.kt

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2023 Slack Technologies, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package slack.cli.shellsentry
+
+import java.nio.file.Path
+
+/**
+ * [ShellSentryExtension] is an extension API to bring your own, complex checkers to [ShellSentry].
+ *
+ * Extensions are given the command, exit code, and console output of the failed command. They can
+ * then process it however they want and return an [AnalysisResult] if they find an issue.
+ *
+ * ## Example
+ *
+ * Below is an example of an extension that asks an AI chat bot to diagnose a failure.
+ *
+ * ```kotlin
+ * public class AiExtension(private val aiClient: AiClient) : ShellSentryExtension {
+ *   override fun check(command: String, exitCode: Int, isAfterRetry: Boolean, consoleOutput: Path): AnalysisResult? {
+ *     val text = consoleOutput.readText()
+ *     val rawAnalysis = aiClient.analyze(text)
+ *     return rawAnalysis.toAnalysisResult()
+ *   }
+ * }
+ * ```
+ */
+public fun interface ShellSentryExtension {
+  /**
+   * Returns a result of this extension's analysis. Returns null if this extension could not handle
+   * failure.
+   *
+   * @param command the command that was executed.
+   * @param exitCode the exit code of the command. Guaranteed to be non-zero.
+   * @param isAfterRetry whether this is after a retry.
+   * @param consoleOutput the path to the console output of the command.
+   * @see AnalysisResult for more details on what goes in a result.
+   */
+  public fun check(
+    command: String,
+    exitCode: Int,
+    isAfterRetry: Boolean,
+    consoleOutput: Path
+  ): AnalysisResult?
+}
+
+/** A returned analysis result in a [ShellSentryExtension]. */
+public data class AnalysisResult(
+  /**
+   * A broad single-line description of the error without specifying exact details, suitable for
+   * crash reporter grouping.
+   */
+  val message: String,
+  /** A detailed, multi-line message explaining the error and suggesting a solution. */
+  val explanation: String,
+  /** A [RetrySignal] indicating if this can be retried. */
+  val retrySignal: RetrySignal,
+  /**
+   * A confidence level, on a scale of [0-100]. This is useful for dynamic analysis that made be
+   * subject to confidence levels, such as an AI analyzer.
+   */
+  val confidence: Int,
+  /**
+   * A function that takes the [message] and returns a [Throwable] for reporting to Bugsnag.
+   * Consider subclassing [NoStacktraceThrowable] if needed.
+   */
+  val throwableMaker: (message: String) -> Throwable
+)