Merge branch 'main' into v2

Author: zegl

README.md

@@ -3,14 +3,17 @@
 [![Discord](https://img.shields.io/badge/join-Discord-blue.svg)](https://discord.gg/nE4UcQHZtV)
 
 
-# CODEBALL — AI CODE REVIEW 🔮
+# 🔮 Codeball — AI Code Review 
 
-Codeball is a code review AI which approves Pull Requests that a human would have approved. Spend less time waiting, save time and money.
+Codeball is a code review AI that scores Pull Requests on a grade from 0 _(needs careful review)_ to 1 _(you should merge this!)_
 
-The AI identifies and approves safe contributions, so that you get to focus your energy on the tricky ones.
+Use Codeball to add labels to help you focus, to auto approve PRs, and more. The Codeball action is easy to use (sane defaults), and is highly customizeable to fit your workflow when needed.
 
-* Identifies and **approves** safe contributions
-* _[beta]_ Generates **code suggestions** from comments ([read more](https://codeball.ai/suggester))
+🏷 Labels PRs when you should **review with caution**  – Stay sharp, don't let the bugs pass through!
+
+✅ Identifies and **approves** or labels safe PRs – Save time by fast-tracking PRs that are easy to review
+
+🏖 **Great defaults**, fully customizable and programmable with GitHub Actions  
 
 ## GitHub Action
 
@@ -38,11 +41,11 @@ jobs:
       - name: Codeball
         uses: sturdy-dev/codeball-action@v2
         with:
+          # For all configuration options see https://github.com/sturdy-dev/codeball-action/blob/v2/action.yml
           approvePullRequests: "true"
           labelPullRequestsWhenApproved: "true"
           labelPullRequestsWhenReviewNeeded: "false"
           failJobsWhenReviewNeeded: "false"
-          codeSuggestionsFromComments: "false" # Beta (set to "true" to enable)
 ```
 
 2. 🎉 That's it! Codeball will now run on new Pull Requests, and will approve the PR if it's a good one!

action.yml

@@ -12,18 +12,37 @@ inputs:
     description: 'If "true", the action will submit an approving review if the Codeball AI approves the contribution'
     default: "true"
     required: false
+
   labelPullRequestsWhenApproved:
-    description: 'If "true", the action will add `codeball:approved` label to the Pull Request if Codeball AI approves the contribution'
+    description: 'If "true", the action will add `codeball:approved` label to the PR if the Codeball AI confidence is above the configured approve threshold'
     default: "true"
     required: false
+
   labelPullRequestsWhenReviewNeeded:
-    description: 'If "true", the action will add `codeball:needs-review` label to the Pull Request if the Codeball AI approves the contribution'
+    description: 'If "true", the action will add `codeball:needs-review` label to the PR if the Codeball AI confidence is between the "approve" and "careful" thresholds'
     default: "false"
     required: false
+
+  labelPullRequestsWhenCarefulReviewNeeded:
+    description: 'If "true", the action will add `codeball:needs-careful-review` label to the PR if the Codeball AI confidence is below the configured careful threshold'
+    default: "true"
+    required: false
+
+  approveThreshold:
+    description: 'The threshold to use for "approving" (greater than or equal to). A number between 0 and 1. Must be specified with 3 decimals.'
+    default: "0.935"
+    required: false
+
+  carefulReviewThreshold:
+    description: 'The threshold to use for "careful review" actions (less than). A number between 0 and 1. Must be specified with 3 decimals.'
+    default: "0.300"
+    required: false
+
   failJobsWhenReviewNeeded:
     description: 'If "true", the action will exit with status code 1 if the Codeball AI does not approve the contribution'
     default: "false"
     required: false
+
   codeSuggestionsFromComments:
     description: 'If "true", Codeball will read generate code suggestions from comments made in Pull Requests (beta)'
     default: "false"
@@ -46,30 +65,43 @@ runs:
       with:
         codeball-job-id: ${{ steps.codeball_baller.outputs.codeball-job-id }}
 
-    # If Codeball approved the contribution, add a "codeball:approved" label
+    # If the Codeball confidence is high, add the "codeball:approved" label
     - name: Label Approved
       uses: sturdy-dev/codeball-action/labeler@v2
-      if: ${{ steps.codeball_status.outputs.approved == 'true' && inputs.labelPullRequestsWhenApproved == 'true' && steps.codeball_status.outputs.jobType == 'contribution' }}
+      if: ${{ steps.codeball_status.outputs.jobType == 'contribution' && inputs.labelPullRequestsWhenApproved == 'true' && steps.codeball_status.outputs.confidence >= inputs.approveThreshold  }}
       with:
         name: "codeball:approved"
         color: "86efac" # green
-        remove-label-names: "codeball:needs-review"
+        remove-label-names: "codeball:needs-review,codeball:needs-careful-review"
         codeball-job-id: ${{ steps.codeball_baller.outputs.codeball-job-id }}
 
-    # If Codeball did not approve the contribution, add a "codeball:needs-review" label
+
+    # If the Codeball confidence is medium, add the "codeball:needs-review" label
     - name: Label Needs Review
       uses: sturdy-dev/codeball-action/labeler@v2
-      if: ${{ steps.codeball_status.outputs.approved == 'false' && inputs.labelPullRequestsWhenReviewNeeded == 'true' && steps.codeball_status.outputs.jobType == 'contribution' }}
+      if: ${{ steps.codeball_status.outputs.jobType == 'contribution' && inputs.labelPullRequestsWhenReviewNeeded == 'true' && steps.codeball_status.outputs.confidence >= inputs.carefulReviewThreshold && steps.codeball_status.outputs.confidence < inputs.approveThreshold  }}
       with:
         name: "codeball:needs-review"
         color: "bfdbfe" # blue
-        remove-label-names: "codeball:approved"
+        remove-label-names: "codeball:approved,codeball:needs-careful-review"
         codeball-job-id: ${{ steps.codeball_baller.outputs.codeball-job-id }}
 
+
+    # If the Codeball confidence is low, add the "codeball:needs-careful-review" label
+    - name: Label Needs Careful Review
+      uses: sturdy-dev/codeball-action/labeler@v2
+      if: ${{ steps.codeball_status.outputs.jobType == 'contribution' && inputs.labelPullRequestsWhenCarefulReviewNeeded == 'true' && steps.codeball_status.outputs.confidence < inputs.carefulReviewThreshold }}
+      with:
+        name: "codeball:needs-careful-review"
+        color: "fde68a" # amber
+        remove-label-names: "codeball:approved,codeball:needs-review"
+        codeball-job-id: ${{ steps.codeball_baller.outputs.codeball-job-id }}
+
+
     # If Codeball approved the contribution, approve the PR
     - name: Approve PR
       uses: sturdy-dev/codeball-action/approver@v2
-      if: ${{ steps.codeball_status.outputs.approved == 'true' && inputs.approvePullRequests == 'true' && steps.codeball_status.outputs.jobType == 'contribution' }}
+      if: ${{ steps.codeball_status.outputs.jobType == 'contribution' && inputs.approvePullRequests == 'true' && steps.codeball_status.outputs.confidence >= inputs.approveThreshold }}
       with:
         codeball-job-id: ${{ steps.codeball_baller.outputs.codeball-job-id }}
 

package.json

@@ -31,23 +31,23 @@
     "@actions/core": "^1.9.1",
     "@actions/github": "^5.0.3",
     "@octokit/core": "^4.0.5",
-    "@octokit/plugin-paginate-rest": "^4.1.0",
-    "@octokit/plugin-rest-endpoint-methods": "^6.3.0",
+    "@octokit/plugin-paginate-rest": "^4.2.0",
+    "@octokit/plugin-rest-endpoint-methods": "^6.4.1",
     "https-proxy-agent": "^5.0.1",
     "node-fetch": "^3.2.10"
   },
   "devDependencies": {
     "@types/jest": "^27.5.1",
     "@types/node": "^18.7.14",
-    "@typescript-eslint/parser": "^5.36.0",
+    "@typescript-eslint/parser": "^5.36.2",
     "@vercel/ncc": "^0.34.0",
-    "eslint": "^8.16.0",
+    "eslint": "^8.23.0",
     "eslint-plugin-github": "^4.3.7",
     "eslint-plugin-jest": "^26.4.0",
     "eslint-plugin-prettier": "^4.0.0",
     "jest": "^28.1.0",
     "js-yaml": "^4.1.0",
-    "prettier": "^2.6.0",
+    "prettier": "^2.7.1",
     "ts-jest": "^28.0.8",
     "typescript": "^4.8.2"
   }

src/lib/jobs/types.ts

@@ -54,9 +54,14 @@ export type Contribution = {
   created_at: string
   merged_at: any
   result: 'inconclusive' | 'approved' | 'not_approved' | null
+  predicted_outcome?: PredictedOutcome
 }
 
 export type Comment = {
   url: string
   suggestions: Suggestion[]
 }
+
+export type PredictedOutcome = {
+  file_probabilities?: number[]
+}

src/status/main.ts

@@ -1,10 +1,10 @@
 import {
+  get,
+  isCommentJob,
   isContributionJob,
   isFinalStatus,
-  get,
-  required,
   Job,
-  isCommentJob
+  required
 } from '../lib'
 import * as core from '@actions/core'
 import {track} from '../lib/track'
@@ -29,9 +29,29 @@ const getJobType = (job: Job): string => {
   return 'unknown'
 }
 
+const getConfidence = (job: Job): number => {
+  if (isContributionJob(job)) {
+    const probabilities =
+      job.contribution?.predicted_outcome?.file_probabilities
+
+    if (probabilities) {
+      return Math.min(...probabilities)
+    }
+
+    return 0
+  }
+
+  return 0
+}
+
 const run = async (
   jobID: string
-): Promise<{isApproved: boolean; isSuggested: boolean; jobType: string}> => {
+): Promise<{
+  isApproved: boolean
+  isSuggested: boolean
+  jobType: string
+  confidence: number
+}> => {
   core.info(`Job ID: ${jobID}`)
 
   let job = await get(jobID)
@@ -49,18 +69,21 @@ const run = async (
   return {
     isApproved: isApproved(job),
     isSuggested: isSuggested(job),
-    jobType: getJobType(job)
+    jobType: getJobType(job),
+    confidence: getConfidence(job)
   }
 }
 
 const jobID = required('codeball-job-id')
 
 run(jobID)
-  .then(async ({isApproved, isSuggested, jobType}) => {
+  .then(async ({isApproved, isSuggested, jobType, confidence}) => {
     await track({jobID, actionName: 'status'})
     core.setOutput('approved', isApproved)
     core.setOutput('suggested', isSuggested)
     core.setOutput('jobType', jobType)
+    core.setOutput('confidence', confidence.toFixed(3))
+    core.info(`Confidence: ${confidence.toFixed(3)}`)
   })
   .catch(async error => {
     if (error instanceof Error) {

status/README.md

@@ -10,10 +10,20 @@ The ID of the Codeball Job created by the Baller Action
 
 # Outputs
 
+## `jobType`
+
+`"contribution"` or `"comment`
+
 ## `approved`
 
 If the Codeball approved the contribution (true or false)
 
 ## `suggested`
 
 If the Codeball has suggestions for the contribution (true or false)
+
+## `confidence`
+
+The Codeball confidence that this contribution can be approved as-is. A number between 0 and 1. A value between 0 and 0.3 normally indicates that the contribution should be thoroughly reviewed. A value above 0.93 indicates that the contribution is very likely to be approved as-is.
+
+Confidence is only set for contribution jobs.

status/action.yml

@@ -17,6 +17,8 @@ outputs:
     description: 'If the Codeball approved the contribution (true or false)'
   suggested:
     description: 'If the Codeball has suggestions for the contribution (true or false)'
+  confidence:
+    description: 'The Codeball confidence that this contribution can be approved as-is. A number between 0 and 1. A value between 0 and 0.3 normally indicates that the contribution should be thoroughly reviewed. A value above 0.93 indicates that the contribution is very likely to be approved as-is. Confidence is only set for contribution jobs.'
 
 runs:
   using: 'node16'