test-pr

README.md

@@ -174,6 +174,121 @@ Java Quarkus REST API that manages [SAST-AI-Workflow](https://github.com/RHEcosy
 **Error Responses:**
 - `404 Not Found` - Batch not found
 
+### MLOps Batch
+
+MLOps batch endpoints enable automated testing of multiple NVRs (Name-Version-Release) fetched from DVC (Data Version Control). These endpoints are independent from regular job batches and designed specifically for ML operations workflows.
+
+#### `POST /api/v1/mlops-batch`
+**Description:** Submit MLOps batch for processing. Fetches NVR list from DVC and creates individual jobs for each NVR with specified MLOps parameters.
+
+**Request Body:**
+```json
+{
+  "testingDataNvrsVersion": "1.0",
+  "promptsVersion": "1.0",
+  "knownNonIssuesVersion": "1.0",
+  "sastAiImage": "quay.io/ecosystem-appeng/sast-ai-workflow:latest",
+  "submittedBy": "[email protected]"
+}
+```
+
+**Fields:**
+- `testingDataNvrsVersion` (required): DVC version tag for fetching test NVRs
+- `promptsVersion` (required): DVC version for prompts configuration
+- `knownNonIssuesVersion` (required): DVC version for known non-issues data
+- `sastAiImage` (required): Container image for SAST AI workflow
+- `submittedBy` (optional): User or system that submitted the batch
+
+**Response:** `201 Created`
+```json
+{
+  "batchId": 123,
+  "testingDataNvrsVersion": "1.0",
+  "promptsVersion": "1.0",
+  "knownNonIssuesVersion": "1.0",
+  "sastAiImage": "quay.io/ecosystem-appeng/sast-ai-workflow:latest",
+  "submittedBy": "[email protected]",
+  "submittedAt": "2025-01-01T10:00:00",
+  "status": "PROCESSING",
+  "totalJobs": 0,
+  "completedJobs": 0,
+  "failedJobs": 0,
+  "lastUpdatedAt": "2025-01-01T10:00:00"
+}
+```
+
+**Error Responses:**
+- `400 Bad Request` - Invalid request body or DVC fetch failure
+
+#### `GET /api/v1/mlops-batch`
+**Description:** List all MLOps batches with pagination
+
+**Query Parameters:**
+- `page` (optional, default: 0): Page number  
+- `size` (optional, default: 20): Page size
+
+**Response:** `200 OK` - Array of MLOps batch objects (same structure as batch creation response)
+
+#### `GET /api/v1/mlops-batch/{batchId}`
+**Description:** Get specific MLOps batch details
+
+**Response:** `200 OK` - Same structure as batch creation response
+
+**Error Responses:**
+- `404 Not Found` - MLOps batch not found
+
+#### `GET /api/v1/mlops-batch/{batchId}/detailed`
+**Description:** Get detailed MLOps batch information including all jobs and their metrics
+
+**Response:** `200 OK`
+```json
+{
+  "batchId": 123,
+  "testingDataNvrsVersion": "1.0",
+  "promptsVersion": "1.0",
+  "knownNonIssuesVersion": "1.0",
+  "sastAiImage": "quay.io/ecosystem-appeng/sast-ai-workflow:latest",
+  "submittedBy": "[email protected]",
+  "submittedAt": "2025-01-01T10:00:00",
+  "status": "COMPLETED_WITH_ERRORS",
+  "totalJobs": 5,
+  "completedJobs": 3,
+  "failedJobs": 2,
+  "lastUpdatedAt": "2025-01-01T11:00:00",
+  "jobs": [
+    {
+      "jobId": 456,
+      "packageNvr": "acl-2.3.2-1.el10",
+      "packageName": "acl",
+      "projectName": "acl",
+      "projectVersion": "2.3.2-1",
+      "packageSourceCodeUrl": "https://download.devel.redhat.com/brewroot/vol/rhel-10/packages/acl/2.3.2/1.el10/src/acl-2.3.2-1.el10.src.rpm",
+      "knownFalsePositivesUrl": "https://gitlab.cee.redhat.com/osh/known-false-positives/-/raw/master/acl/ignore.err",
+      "status": "COMPLETED",
+      "createdAt": "2025-01-01T10:00:00",
+      "startedAt": "2025-01-01T10:01:00",
+      "completedAt": "2025-01-01T10:30:00",
+      "tektonUrl": "https://tekton.example.com/pipelineruns/job-456",
+      "metrics": {
+        "accuracy": 0.95,
+        "precision": 0.87,
+        "recall": 0.92,
+        "f1Score": 0.89,
+        "confusionMatrix": {
+          "tp": 10,
+          "fp": 2,
+          "tn": 8,
+          "fn": 1
+        }
+      }
+    }
+  ]
+}
+```
+
+**Error Responses:**
+- `404 Not Found` - MLOps batch not found
+
 ### Package Analysis
 
 #### `GET /api/v1/packages`

deploy/sast-ai-chart/templates/deployment.yaml

@@ -54,6 +54,22 @@ spec:
                 secretKeyRef:
                   name: {{ include "sast-ai.postgresql.secretName" . }}
                   key: {{ include "sast-ai.postgresql.secretKey" . }}
+            # AWS S3 credentials for DVC (MinIO)
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: sast-ai-s3-credentials
+                  key: access_key_id
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: sast-ai-s3-credentials
+                  key: secret_access_key
+            - name: AWS_S3_ENDPOINT_URL
+              valueFrom:
+                secretKeyRef:
+                  name: sast-ai-s3-credentials
+                  key: endpoint_url
             # Application environment variables
             {{- range $key, $value := .Values.app.env }}
             - name: {{ $key }}

pom.xml

@@ -166,11 +166,23 @@
             <artifactId>quarkus-jacoco</artifactId>
             <scope>test</scope>
         </dependency>
+        <!-- YAML parser for DVC YAML files -->
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>2.3</version>
+        </dependency>
         <dependency>
             <groupId>org.jsoup</groupId>
             <artifactId>jsoup</artifactId>
             <version>1.18.3</version>
         </dependency>
+        <!-- YAML parser for DVC YAML files -->
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>2.5</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/docker/Dockerfile.jvm

@@ -82,6 +82,14 @@ FROM registry.access.redhat.com/ubi9/openjdk-21:1.21
 
 ENV LANGUAGE='en_US:en'
 
+# Install DVC CLI and dependencies for data version control
+USER 0
+RUN microdnf install -y python3 python3-pip git && \
+    pip3 install --no-cache-dir dvc dvc-s3 && \
+    microdnf clean all && \
+    chmod -R 755 /usr/local/bin/dvc* && \
+    chmod -R 755 /usr/local/lib*/python*/site-packages/dvc* || true
+USER 185
 
 # We make four distinct layers so if there are application changes the library layers can be re-used
 COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/

src/main/java/com/redhat/sast/api/exceptions/DvcException.java

@@ -0,0 +1,28 @@
+package com.redhat.sast.api.exceptions;
+
+/**
+ * Exception thrown when DVC operations fail.
+ * This includes failures in fetching data from DVC repositories,
+ * parsing DVC YAML files, or DVC command execution errors.
+ */
+public class DvcException extends RuntimeException {
+
+    /**
+     * Constructs a new DvcException with the specified detail message.
+     *
+     * @param message the detail message
+     */
+    public DvcException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructs a new DvcException with the specified detail message and cause.
+     *
+     * @param message the detail message
+     * @param cause the cause of this exception
+     */
+    public DvcException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/redhat/sast/api/model/JobBatch.java

@@ -12,9 +12,7 @@
 import lombok.NoArgsConstructor;
 
 @Entity
-@Table(
-        name = "job_batch",
-        indexes = {@Index(name = "idx_job_batch_id", columnList = "id")})
+@Table(name = "job_batch")
 @Data
 @NoArgsConstructor
 @EqualsAndHashCode(exclude = {"jobs", "jobBatchExecutionContext", "jobBatchRunDefinition"})

src/main/java/com/redhat/sast/api/model/MlOpsBatch.java

@@ -0,0 +1,88 @@
+package com.redhat.sast.api.model;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.redhat.sast.api.enums.BatchStatus;
+
+import jakarta.persistence.*;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+@Entity
+@Table(name = "mlops_batch")
+@Data
+@NoArgsConstructor
+@EqualsAndHashCode(exclude = {"jobs"})
+public class MlOpsBatch {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private Long id;
+
+    @Column(name = "testing_data_nvrs_version", nullable = false, length = 100)
+    private String testingDataNvrsVersion;
+
+    @Column(name = "prompts_version", nullable = false, length = 100)
+    private String promptsVersion;
+
+    @Column(name = "known_non_issues_version", nullable = false, length = 100)
+    private String knownNonIssuesVersion;
+
+    @Column(name = "container_image", nullable = false, length = 500)
+    private String containerImage;
+
+    @Column(name = "submitted_by")
+    private String submittedBy;
+
+    @Column(name = "submitted_at", nullable = false)
+    private LocalDateTime submittedAt;
+
+    @Enumerated(EnumType.STRING)
+    @Column(name = "status", nullable = false)
+    private BatchStatus status;
+
+    @Column(name = "total_jobs")
+    private Integer totalJobs;
+
+    @Column(name = "completed_jobs")
+    private Integer completedJobs;
+
+    @Column(name = "failed_jobs")
+    private Integer failedJobs;
+
+    @Column(name = "last_updated_at")
+    private LocalDateTime lastUpdatedAt;
+
+    @OneToMany(mappedBy = "mlOpsBatch", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
+    private List<MlOpsJob> jobs;
+
+    @PrePersist
+    public void prePersist() {
+        this.submittedAt = LocalDateTime.now();
+        this.lastUpdatedAt = LocalDateTime.now();
+        this.status = BatchStatus.PROCESSING;
+        this.totalJobs = 0;
+        this.completedJobs = 0;
+        this.failedJobs = 0;
+    }
+
+    @PreUpdate
+    public void preUpdate() {
+        this.lastUpdatedAt = LocalDateTime.now();
+    }
+
+    public List<MlOpsJob> getJobs() {
+        if (jobs == null) {
+            jobs = new ArrayList<>();
+        }
+        return jobs;
+    }
+
+    public void setJobs(List<MlOpsJob> jobs) {
+        this.jobs = jobs != null ? new ArrayList<>(jobs) : new ArrayList<>();
+    }
+}