Adds a new task to detect CSAM using an external provider

iped-app/resources/config/IPEDConfig.txt

@@ -171,3 +171,8 @@ enableHTMLReport = true
 # It performs classification of image and video files.
 # Advanced settings can be modified in file "conf/RemoteImageClassifierConfig.txt"
 enableRemoteImageClassifier = false
+
+# Enables remote external image classifier.
+# It performs classification of image and video files using review priority and embeddings output.
+# Advanced settings can be modified in file "conf/RemoteImageExternalClassifierConfig.txt"
+enableRemoteImageExternalClassifier = true

iped-app/resources/config/conf/AIFiltersConfig.json

@@ -19,6 +19,14 @@
         {"name": "Other"}
     ]},
 
+    {"name": "External CSAM Detector", "prefix": "ExternalCSAM", "property": "ai\\:externalClassifier\\:reviewPriority", "value": "*", "children":[
+        {"name": "Very High", "value": "VERY_HIGH"},
+        {"name": "High", "value": "HIGH"},
+        {"name": "Medium", "value": "MEDIUM"},
+        {"name": "Low", "value": "LOW"},
+        {"name": "Very Low", "value": "VERY_LOW"}
+    ]},
+
     {"name": "Detected Faces", "prefix": "Face", "property": "face\\_count", "value": "[1 TO *]", "children":[
         {"name": "1"},
         {"name": "2"},

iped-app/resources/config/conf/RemoteImageExternalClassifierConfig.txt

@@ -0,0 +1,25 @@
+##########################################
+# RemoteImageExternalClassifier Task configuration
+##########################################
+
+# URL of the service/central node used by the RemoteImageExternalClassifier implementation
+url = http://localhost:8000
+
+# Maximum number of thumbs to be included in the zip file to send to the server
+batchSize = 50
+
+# Optional: used only if backend also returns legacy "class" probabilities.
+# Minimum confidence threshold for assigning a class label to an image/video. Values are between 0 and 100.
+labelingThreshold = 50
+
+# Skip classification of images/videos smaller than a given file size (in bytes; '0' = do not skip)
+skipSize = 2048
+
+# Skip classification of images/videos smaller than a given dimension, i.e. height or width (in pixels; '0' = do not skip)
+skipDimension = 48
+
+# Skip classification of images/videos with hits on IPED hashesDB database (if 'hashesDB' is not configured in 'LocalConfig.txt' or 'false', do not skip)
+skipHashDBFiles = true
+
+# Validate server SSL certificate
+validateSSL = false

iped-app/resources/config/conf/TaskInstaller.xml

@@ -40,6 +40,7 @@
     <task class="iped.engine.task.PhotoDNATask"></task>
     <task class="iped.engine.task.PhotoDNALookup"></task>
     <task class="iped.engine.task.RemoteImageClassifierTask"></task>
+    <task class="iped.engine.task.RemoteImageExternalClassifierTask"></task>
     <task script="NSFWNudityDetectTask.py"></task>
     <task script="FaceRecognitionTask.py"></task>
     <task script="AgeEstimationTask.py"></task>

iped-engine/src/main/java/iped/engine/config/RemoteImageExternalClassifierConfig.java

@@ -0,0 +1,137 @@
+package iped.engine.config;
+
+import iped.utils.UTF8Properties;
+
+public class RemoteImageExternalClassifierConfig extends AbstractTaskPropertiesConfig {
+
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Config file name and enable/disable property.
+     */
+    private static final String CONFIG_FILE = "RemoteImageExternalClassifierConfig.txt";
+    private static final String ENABLE_PROP = "enableRemoteImageExternalClassifier";
+
+    /**
+     * Constants mapping to properties name in config file.
+     */
+    private static final String URL = "url";
+    private static final String BATCH_SIZE = "batchSize";
+    private static final String LABELING_THRESHOLD = "labelingThreshold";
+    private static final String SKIP_SIZE = "skipSize";
+    private static final String SKIP_DIMENSION = "skipDimension";
+    private static final String SKIP_HASH_DB_FILES = "skipHashDBFiles";
+    private static final String VALIDATE_SSL = "validateSSL";
+
+    // URL of the service/central node used by the RemoteImageClassifier implementation
+    private String url;
+
+    // Maximum number of thumbs to be included in the zip file to send to the server
+    private int batchSize = 50;
+
+    // Threshold used to decide if an image is labeled with one class
+    private double labelingThreshold = 60;
+
+    // Skip classification of images/videos smaller than a given file size (in bytes; '0' = do not skip)
+    private int skipSize = 0;
+
+    // Skip classification of images/videos smaller than a given dimension, i.e. height or width (in pixels; '0' = do not skip)
+    private int skipDimension = 0;
+
+    // Skip classification of images/videos with hits on IPED hashesDB database (if 'hashesDB' is not configured in 'LocalConfig.txt' or 'false', do not skip)
+    private boolean skipHashDBFiles = true;   
+
+    // Validate server SSL certificate
+    private boolean validateSSL = false;
+
+    @Override
+    public String getTaskEnableProperty() {
+        return ENABLE_PROP;
+    }
+
+    @Override
+    public String getTaskConfigFileName() {
+        return CONFIG_FILE;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public int getBatchSize() {
+        return batchSize;
+    }
+
+    public double getLabelingThreshold() {
+        return labelingThreshold;
+    }
+
+    public int getSkipSize() {
+        return skipSize;
+    }
+
+    public int getSkipDimension() {
+        return skipDimension;
+    }
+
+    public boolean isSkipHashDBFiles() {
+        return skipHashDBFiles;
+    }
+
+    public boolean isValidateSSL() {
+        return validateSSL;
+    }
+
+    @Override
+    void processProperties(UTF8Properties properties) {
+
+        String value = properties.getProperty(URL);
+        if (value != null && !value.trim().isEmpty())
+            url = value.trim();
+
+        value = properties.getProperty(BATCH_SIZE);
+        if (value != null && !value.trim().isEmpty()) {
+            batchSize = Integer.valueOf(value.trim());
+            // enforce minimum value
+            if (batchSize < 1)
+                batchSize = 1;
+        }
+
+        value = properties.getProperty(LABELING_THRESHOLD);
+        if (value != null && !value.trim().isEmpty()) {
+            labelingThreshold = Double.parseDouble(value.trim());
+            // enforce range [0, 100]
+            if (labelingThreshold < 0) {
+                labelingThreshold = 0;
+            } else if (labelingThreshold > 100) {
+                labelingThreshold = 100;
+            }
+        }
+
+        value = properties.getProperty(SKIP_SIZE);
+        if (value != null && !value.trim().isEmpty()) {
+            skipSize = Integer.valueOf(value.trim());
+            // enforce minimum value
+            if (skipSize < 0)
+                skipSize = 0;
+        }
+
+        value = properties.getProperty(SKIP_DIMENSION);
+        if (value != null && !value.trim().isEmpty()) {
+            skipDimension = Integer.valueOf(value.trim());
+            // enforce minimum value
+            if (skipDimension < 0)
+                skipDimension = 0;
+        }
+
+        value = properties.getProperty(SKIP_HASH_DB_FILES);
+        if (value != null && !value.trim().isEmpty())
+            skipHashDBFiles = Boolean.valueOf(value.trim());
+
+        value = properties.getProperty(VALIDATE_SSL);
+        if (value != null && !value.trim().isEmpty())
+            validateSSL = Boolean.valueOf(value.trim());
+
+    }
+
+}

iped-engine/src/main/java/iped/engine/lucene/analysis/AppAnalyzer.java

@@ -29,6 +29,7 @@
 import iped.engine.config.IndexTaskConfig;
 import iped.engine.task.HashTask;
 import iped.engine.task.PhotoDNATask;
+import iped.engine.task.RemoteImageExternalClassifierTask;
 import iped.engine.task.index.IndexItem;
 import iped.localization.LocalizedProperties;
 import iped.properties.ExtraProperties;
@@ -57,6 +58,8 @@ public static Analyzer get() {
         analyzerPerField.put(IndexItem.CHANGED, new KeywordAnalyzer());
         analyzerPerField.put(IndexItem.TIMESTAMP, new KeywordAnalyzer());
 
+        analyzerPerField.put(RemoteImageExternalClassifierTask.AI_REVIEW_PRIORITY_ATTR, new KeywordAnalyzer());
+
         IndexTaskConfig indexConfig = ConfigurationManager.get().findObject(IndexTaskConfig.class);
         StandardASCIIAnalyzer hashAnalyzer = new StandardASCIIAnalyzer();
         hashAnalyzer.setMaxTokenLength(Integer.MAX_VALUE);