[backend] Add collector state (#3329)

Signed-off-by: Marine LM <[email protected]>
Co-authored-by: Romuald Lemesle <[email protected]>

Author: MarineLeM

openbas-api/src/main/java/io/openbas/collectors/expectations_expiration_manager/ExpectationsExpirationManagerCollector.java

@@ -2,7 +2,7 @@
 
 import io.openbas.collectors.expectations_expiration_manager.config.ExpectationsExpirationManagerConfig;
 import io.openbas.collectors.expectations_expiration_manager.service.ExpectationsExpirationManagerService;
-import io.openbas.integrations.CollectorService;
+import io.openbas.rest.collector.service.CollectorService;
 import jakarta.annotation.PostConstruct;
 import java.time.Duration;
 import lombok.RequiredArgsConstructor;

openbas-api/src/main/java/io/openbas/collectors/expectations_expiration_manager/ExpectationsExpirationManagerJob.java

@@ -2,7 +2,7 @@
 
 import io.openbas.collectors.expectations_expiration_manager.config.ExpectationsExpirationManagerConfig;
 import io.openbas.collectors.expectations_expiration_manager.service.ExpectationsExpirationManagerService;
-import io.openbas.integrations.CollectorService;
+import io.openbas.rest.collector.service.CollectorService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;

openbas-api/src/main/java/io/openbas/collectors/expectations_vulnerability_manager/ExpectationsVulnerabilityManagerCollector.java

@@ -1,6 +1,6 @@
 package io.openbas.collectors.expectations_vulnerability_manager;
 
-import io.openbas.integrations.CollectorService;
+import io.openbas.rest.collector.service.CollectorService;
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

openbas-api/src/main/java/io/openbas/migration/V4_14__Add_collector_state.java

@@ -0,0 +1,22 @@
+package io.openbas.migration;
+
+import java.sql.Statement;
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.springframework.stereotype.Component;
+
+@Component
+public class V4_14__Add_collector_state extends BaseJavaMigration {
+
+  @Override
+  public void migrate(Context context) throws Exception {
+    Statement select = context.getConnection().createStatement();
+    select.execute(
+        """
+                  ALTER TABLE collectors
+                    ADD COLUMN collector_state JSONB DEFAULT '{}' ;
+                  """);
+
+    select.execute("ALTER TABLE cves RENAME COLUMN cve_cvss TO cve_cvss_v31;");
+  }
+}

openbas-api/src/main/java/io/openbas/rest/collector/CollectorApi.java

@@ -2,54 +2,33 @@
 
 import static io.openbas.database.model.User.ROLE_ADMIN;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import io.openbas.config.OpenBASConfig;
 import io.openbas.database.model.Collector;
 import io.openbas.database.repository.CollectorRepository;
 import io.openbas.database.repository.SecurityPlatformRepository;
 import io.openbas.rest.collector.form.CollectorCreateInput;
 import io.openbas.rest.collector.form.CollectorUpdateInput;
-import io.openbas.rest.exception.ElementNotFoundException;
+import io.openbas.rest.collector.service.CollectorService;
 import io.openbas.rest.helper.RestBehavior;
 import io.openbas.service.FileService;
-import jakarta.annotation.Resource;
 import jakarta.transaction.Transactional;
 import jakarta.validation.Valid;
 import java.time.Instant;
 import java.util.Optional;
-import org.springframework.beans.factory.annotation.Autowired;
+import lombok.RequiredArgsConstructor;
 import org.springframework.http.MediaType;
 import org.springframework.security.access.annotation.Secured;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
 @RestController
+@RequiredArgsConstructor
 public class CollectorApi extends RestBehavior {
 
-  @Resource private OpenBASConfig openBASConfig;
+  private final CollectorService collectorService;
+  private final CollectorRepository collectorRepository;
+  private final SecurityPlatformRepository securityPlatformRepository;
 
-  private CollectorRepository collectorRepository;
-
-  private FileService fileService;
-
-  private SecurityPlatformRepository securityPlatformRepository;
-
-  @Resource protected ObjectMapper mapper;
-
-  @Autowired
-  public void setFileService(FileService fileService) {
-    this.fileService = fileService;
-  }
-
-  @Autowired
-  public void setCollectorRepository(CollectorRepository collectorRepository) {
-    this.collectorRepository = collectorRepository;
-  }
-
-  @Autowired
-  public void setSecurityPlatformRepository(SecurityPlatformRepository securityPlatformRepository) {
-    this.securityPlatformRepository = securityPlatformRepository;
-  }
+  private final FileService fileService;
 
   @GetMapping("/api/collectors")
   public Iterable<Collector> collectors() {
@@ -76,13 +55,18 @@ private Collector updateCollector(
     return collectorRepository.save(collector);
   }
 
+  @GetMapping("/api/collectors/{collectorId}")
+  @Secured(ROLE_ADMIN)
+  public Collector getCollector(@PathVariable String collectorId) {
+    return collectorService.collector(collectorId);
+  }
+
   @Secured(ROLE_ADMIN)
   @PutMapping("/api/collectors/{collectorId}")
   @Transactional(rollbackOn = Exception.class)
   public Collector updateCollector(
       @PathVariable String collectorId, @Valid @RequestBody CollectorUpdateInput input) {
-    Collector collector =
-        collectorRepository.findById(collectorId).orElseThrow(ElementNotFoundException::new);
+    Collector collector = collectorService.collector(collectorId);
     return updateCollector(
         collector,
         collector.getType(),

openbas-framework/src/main/java/io/openbas/integrations/CollectorService.java renamed to openbas-api/src/main/java/io/openbas/rest/collector/service/CollectorService.java

@@ -1,37 +1,50 @@
-package io.openbas.integrations;
+package io.openbas.rest.collector.service;
 
 import static io.openbas.service.FileService.COLLECTORS_IMAGES_BASE_PATH;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.openbas.database.model.Collector;
 import io.openbas.database.repository.CollectorRepository;
+import io.openbas.rest.exception.ElementNotFoundException;
 import io.openbas.service.FileService;
 import jakarta.annotation.Resource;
 import jakarta.transaction.Transactional;
-import jakarta.validation.constraints.NotBlank;
 import java.io.InputStream;
-import org.springframework.beans.factory.annotation.Autowired;
+import java.util.Optional;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
+@Slf4j
 @Service
+@RequiredArgsConstructor
 public class CollectorService {
 
   @Resource protected ObjectMapper mapper;
 
-  private FileService fileService;
+  private final CollectorRepository collectorRepository;
+  private final FileService fileService;
 
-  private CollectorRepository collectorRepository;
+  // -- CRUD --
 
-  @Resource
-  public void setFileService(FileService fileService) {
-    this.fileService = fileService;
+  public Collector collector(String id) {
+    return collectorRepository
+        .findById(id)
+        .orElseThrow(() -> new ElementNotFoundException("Collector not found with id: " + id));
   }
 
-  @Autowired
-  public void setCollectorRepository(CollectorRepository collectorRepository) {
-    this.collectorRepository = collectorRepository;
+  public Collector updateCollectorState(Collector collectorToUpdate, ObjectNode newState) {
+    ObjectNode state =
+        Optional.ofNullable(collectorToUpdate.getState()).orElse(mapper.createObjectNode());
+    newState
+        .fieldNames()
+        .forEachRemaining(fieldName -> state.set(fieldName, newState.get(fieldName)));
+    return collectorRepository.save(collectorToUpdate);
   }
 
+  // -- ACTION --
+
   @Transactional
   public void register(String id, String type, String name, InputStream iconData) throws Exception {
     if (iconData != null) {
@@ -61,12 +74,4 @@ public void register(String id, String type, String name, InputStream iconData)
       collectorRepository.save(newCollector);
     }
   }
-
-  // -- CRUD --
-
-  public Collector collector(@NotBlank final String collectorId) {
-    return collectorRepository
-        .findById(collectorId)
-        .orElseThrow(() -> new RuntimeException("Collector not found"));
-  }
 }

openbas-api/src/main/java/io/openbas/rest/cve/CveApi.java

@@ -4,10 +4,7 @@
 import static io.openbas.database.model.User.ROLE_USER;
 
 import io.openbas.aop.LogExecutionTime;
-import io.openbas.rest.cve.form.CveCreateInput;
-import io.openbas.rest.cve.form.CveOutput;
-import io.openbas.rest.cve.form.CveSimple;
-import io.openbas.rest.cve.form.CveUpdateInput;
+import io.openbas.rest.cve.form.*;
 import io.openbas.rest.cve.service.CveService;
 import io.openbas.rest.helper.RestBehavior;
 import io.openbas.utils.mapper.CveMapper;
@@ -16,6 +13,7 @@
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jakarta.transaction.Transactional;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotNull;
 import lombok.RequiredArgsConstructor;
 import org.springframework.data.domain.Page;
 import org.springframework.security.access.annotation.Secured;
@@ -61,6 +59,14 @@ public CveSimple createCve(@Valid @RequestBody CveCreateInput input) {
     return cveMapper.toCveSimple(cveService.createCve(input));
   }
 
+  @Secured(ROLE_ADMIN)
+  @Operation(summary = "Bulk insert CVEs")
+  @LogExecutionTime
+  @PostMapping(CVE_API + "/bulk")
+  public void bulkInsertCVEsForCollector(@Valid @RequestBody @NotNull CVEBulkInsertInput input) {
+    this.cveService.bulkUpsertCVEs(input);
+  }
+
   @Secured(ROLE_ADMIN)
   @Operation(summary = "Update an existing CVE")
   @PutMapping(CVE_API + "/{cveId}")

openbas-api/src/main/java/io/openbas/rest/cve/form/CVEBulkInsertInput.java

@@ -0,0 +1,29 @@
+package io.openbas.rest.cve.form;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.openbas.config.AppConfig;
+import jakarta.validation.constraints.NotNull;
+import java.time.Instant;
+import java.util.List;
+import lombok.Data;
+
+@Data
+public class CVEBulkInsertInput {
+
+  @JsonProperty("cves")
+  @NotNull(message = AppConfig.MANDATORY_MESSAGE)
+  private List<CveCreateInput> cves;
+
+  @JsonProperty("last_modified_date_fetched")
+  private Instant lastModifiedDateFetched;
+
+  @JsonProperty("last_index")
+  private Integer lastIndex;
+
+  @JsonProperty("initial_dataset_completed")
+  private Boolean initialDatasetCompleted;
+
+  @JsonProperty("source_identifier")
+  @NotNull(message = AppConfig.MANDATORY_MESSAGE)
+  private String sourceIdentifier;
+}

openbas-api/src/main/java/io/openbas/rest/cve/form/CveCreateInput.java

@@ -23,7 +23,7 @@ public class CveCreateInput extends CveInput {
   @NotNull
   @DecimalMin("0.0")
   @DecimalMax("10.0")
-  @JsonProperty("cve_cvss")
+  @JsonProperty("cve_cvss_v31")
   @Schema(description = "CVSS score", example = "7.5", minimum = "0.0", maximum = "10.0")
-  private BigDecimal cvss;
+  private BigDecimal cvssV31;
 }

openbas-api/src/main/java/io/openbas/rest/cve/form/CveSimple.java

@@ -27,9 +27,9 @@ public class CveSimple {
   private String externalId;
 
   @NotNull
-  @JsonProperty("cve_cvss")
+  @JsonProperty("cve_cvss_v31")
   @Schema(description = "CVSS score", example = "7.8")
-  private BigDecimal cvss;
+  private BigDecimal cvssV31;
 
   @JsonProperty("cve_published")
   @Schema(description = "CVE published date")