Merge pull request #81 from AutomateThePlanet/navramov_MM

n1xan · web-flow · commit ee60cda70006 · 2025-08-18T15:00:18.000+03:00
add precision threshold and grayscale options for OpenCV Service
diff --git a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java
@@ -32,7 +32,7 @@ public Point getLocation() {
 
         var encodedImage = findStrategy.getEncodedImage();
 
-        var location = OpenCvService.getLocation(encodedImage, true);
+        var location = OpenCvService.getLocation(encodedImage);
         return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset());
     }
 
diff --git a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java
@@ -16,14 +16,11 @@
 import com.microsoft.playwright.Locator;
 import com.microsoft.playwright.Page;
 import lombok.Getter;
-import solutions.bellatrix.core.utilities.SingletonFactory;
 import solutions.bellatrix.core.utilities.parsing.TypeParser;
 import solutions.bellatrix.playwright.components.common.webelement.WebElement;
-import solutions.bellatrix.playwright.services.JavaScriptService;
 import solutions.bellatrix.plugins.opencv.Base64Encodable;
 import solutions.bellatrix.plugins.opencv.OpenCvService;
 
-import java.awt.*;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -37,7 +34,7 @@ public ImageBase64FindStrategy(Base64Encodable encodedImage) {
 
     @Override
     public WebElement convert(Page page) {
-        var location = OpenCvService.getLocation(encodedImage, false);
+        var location = OpenCvService.getLocation(encodedImage);
 
         var foundLocators = findElementsOn(location, page);
 
@@ -48,7 +45,7 @@ public WebElement convert(Page page) {
 
     @Override
     public WebElement convert(WebElement webElement) {
-        var location = OpenCvService.getLocation(encodedImage, false);
+        var location = OpenCvService.getLocation(encodedImage);
 
         var foundLocators = findElementsOn(location, webElement.page());
 
diff --git a/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java
@@ -8,6 +8,8 @@
 import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.imgproc.Imgproc;
 import plugins.screenshots.ScreenshotPlugin;
+import solutions.bellatrix.core.configuration.ConfigurationService;
+import solutions.bellatrix.core.utilities.Log;
 import solutions.bellatrix.core.utilities.SingletonFactory;
 
 import javax.imageio.ImageIO;
@@ -39,16 +41,24 @@ public static double getJavaMonitorScaling() {
         return scaleX;
     }
 
+    public static Point getLocation(Base64Encodable encodedImage) {
+        OpenCvServiceSettings openCvServiceSettings = ConfigurationService.get(OpenCvServiceSettings.class);
+        if (openCvServiceSettings == null) {
+            openCvServiceSettings = new OpenCvServiceSettings();
+        }
+
+        var precisionThreshold = openCvServiceSettings.getDefaultMatchThreshold();
+        var shouldGrayscale = openCvServiceSettings.isShouldGrayscale();
+        return getLocation(encodedImage, shouldGrayscale, precisionThreshold);
+    }
     /**
      * @return the coordinates of the image found on the screen
      */
-    public static Point getLocation(Base64Encodable encodedImage, boolean shouldGrayScale) {
+    public static Point getLocation(Base64Encodable encodedImage, boolean shouldGrayScale, double precisionThreshold) {
         var screenshotPlugin = SingletonFactory.getInstance(ScreenshotPlugin.class);
-
+        Log.info("Locating image using precision: %s; Should Grayscale = %s".formatted(precisionThreshold, shouldGrayScale));
         if (screenshotPlugin == null) {
-            throw new IllegalArgumentException("It seems that the screenshot plugin isn't registered by the 'ScreenshotPlugin.class' key inside SingletonFactory's map or isn't registered at all!\n" +
-                    "Check the BaseTest class of your project where the plugins are registered. Register the specific screenshot plugin implementation as the base ScreenshotPlugin.class.\n" +
-                    "for example: addPluginAs(ScreenshotPlugin.class, WebScreenshotPlugin.class);");
+            throw new IllegalArgumentException("Screenshot plugin not registered!");
         }
 
         var screenshot = screenshotPlugin.takeScreenshot();
@@ -57,13 +67,17 @@ public static Point getLocation(Base64Encodable encodedImage, boolean shouldGray
 
         Mat result = loadImages(encodedImage, screenshot, shouldGrayScale);
 
-        return getMatchLocation(encodedImage, result);
+        return getMatchLocation(encodedImage, result, precisionThreshold);
     }
 
-    private static Point getMatchLocation(Base64Encodable encodedImage, Mat result) {
+    private static Point getMatchLocation(Base64Encodable encodedImage, Mat result, double precisionThreshold) {
         BufferedImage bufferedImage;
-
         Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
+        if (mmr.maxVal < precisionThreshold) {
+            throw new RuntimeException("Match not found above threshold. Closest match at: %s".formatted(mmr.maxVal));
+        }
+        Log.info("Image located with precision: %s".formatted(mmr.maxVal));
+
         Point matchLoc = mmr.maxLoc;
 
         if (encodedImage.getXOffset() == 0 && encodedImage.getYOffset() == 0) {
@@ -73,13 +87,17 @@ private static Point getMatchLocation(Base64Encodable encodedImage, Mat result)
                 throw new RuntimeException(e);
             }
 
-            double[] imageCenterCoordinates = {matchLoc.x / getJavaMonitorScaling() + (double)(bufferedImage.getWidth() / 2), matchLoc.y / getJavaMonitorScaling() + (double)(bufferedImage.getHeight() / 2)};
+            double[] imageCenterCoordinates = {
+                    matchLoc.x / getJavaMonitorScaling() + (double)(bufferedImage.getWidth() / 2),
+                    matchLoc.y / getJavaMonitorScaling() + (double)(bufferedImage.getHeight() / 2)
+            };
             matchLoc.set(imageCenterCoordinates);
         }
 
         return matchLoc;
     }
 
+
     private static BufferedImage getImageWidthHeight(Base64Encodable encodedImage) throws IOException {
         String cleanBase64 = removePrefixFromBase64(encodedImage);
 
diff --git a/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvServiceSettings.java b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvServiceSettings.java
@@ -0,0 +1,12 @@
+package solutions.bellatrix.plugins.opencv;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
+
+@Getter
+public class OpenCvServiceSettings {
+    @SerializedName("defaultMatchThreshold")
+    private double defaultMatchThreshold = 0.8;
+    @SerializedName("shouldGrayscale")
+    private boolean shouldGrayscale = true;
+}
diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java
@@ -30,7 +30,7 @@ public Point getLocation() {
 
         var encodedImage = findStrategy.getEncodedImage();
 
-        var location = OpenCvService.getLocation(encodedImage, true);
+        var location = OpenCvService.getLocation(encodedImage);
         return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset());
     }
 
diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java
@@ -4,23 +4,39 @@
 import org.openqa.selenium.By;
 import org.openqa.selenium.SearchContext;
 import org.openqa.selenium.WebElement;
+import solutions.bellatrix.core.configuration.ConfigurationService;
 import solutions.bellatrix.core.utilities.Log;
 import solutions.bellatrix.core.utilities.SingletonFactory;
 import solutions.bellatrix.plugins.opencv.Base64Encodable;
 import solutions.bellatrix.plugins.opencv.OpenCvService;
-import solutions.bellatrix.web.services.App;
+import solutions.bellatrix.plugins.opencv.OpenCvServiceSettings;
 import solutions.bellatrix.web.services.JavaScriptService;
 
 import java.util.List;
-import java.util.Objects;
 
 @Getter
 public class ImageBase64FindStrategy extends FindStrategy {
     private final Base64Encodable encodedImage;
+    private final boolean shouldGrayScale;
+    private final double precisionThreshold;
 
     public ImageBase64FindStrategy(Base64Encodable encodedImage) {
         super(encodedImage.getBase64Image());
         this.encodedImage = encodedImage;
+        var serviceSettings = ConfigurationService.get(OpenCvServiceSettings.class);
+        if (serviceSettings == null) {
+            serviceSettings = new OpenCvServiceSettings();
+        }
+
+        this.shouldGrayScale = serviceSettings.isShouldGrayscale();
+        this.precisionThreshold = serviceSettings.getDefaultMatchThreshold();
+    }
+
+    public ImageBase64FindStrategy(Base64Encodable encodedImage, boolean shouldGrayScale, double precisionThreshold) {
+        super(encodedImage.getBase64Image());
+        this.shouldGrayScale = shouldGrayScale;
+        this.precisionThreshold = precisionThreshold;
+        this.encodedImage = encodedImage;
     }
 
     @Override
@@ -46,7 +62,7 @@ public static By byImageBase64(Base64Encodable encodedImage) {
 
         @Override
         public List<WebElement> findElements(SearchContext context) {
-            var location = OpenCvService.getLocation(base64EncodedImage, false);
+            var location = OpenCvService.getLocation(base64EncodedImage);
             Log.info("Coordinates located: %s", location.toString());
             return SingletonFactory.getInstance(JavaScriptService.class).<List<WebElement>>genericExecute("return document.elementsFromPoint(%s, %s);".formatted(location.x, location.y));
         }
diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java
@@ -15,10 +15,8 @@
 
 import solutions.bellatrix.core.utilities.InstanceFactory;
 import solutions.bellatrix.plugins.opencv.Base64Encodable;
-import solutions.bellatrix.web.components.ActionImage;
 import solutions.bellatrix.web.components.WebComponent;
 import solutions.bellatrix.web.findstrategies.*;
-import solutions.bellatrix.web.infrastructure.DriverService;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -42,6 +40,10 @@ public <TComponent extends WebComponent> TComponent byImage(Class<TComponent> co
         return by(componentClass, new ImageBase64FindStrategy(encodedImage));
     }
 
+    public <TComponent extends WebComponent> TComponent byImage(Class<TComponent> componentClass, Base64Encodable encodedImage, boolean shouldGrayscale, double matchPrecision) {
+        return by(componentClass, new ImageBase64FindStrategy(encodedImage, shouldGrayscale, matchPrecision));
+    }
+
     public <TComponent extends WebComponent> TComponent byAttributeContaining(Class<TComponent> componentClass, String attributeName, String value) {
         return by(componentClass, new AttributeContainingWithFindStrategy(attributeName, value));
     }
@@ -106,6 +108,10 @@ public <TComponent extends WebComponent> List<TComponent> allByImage(Class<TComp
         return allBy(componentClass, new ImageBase64FindStrategy(encodedImage));
     }
 
+    public <TComponent extends WebComponent> List<TComponent> allByImage(Class<TComponent> componentClass, Base64Encodable encodedImage, boolean shouldGrayscale, double matchPrecision) {
+        return allBy(componentClass, new ImageBase64FindStrategy(encodedImage, shouldGrayscale, matchPrecision));
+    }
+
     public <TComponent extends WebComponent> List<TComponent> allByAttributeContaining(Class<TComponent> componentClass, String attributeName, String value) {
         return allBy(componentClass, new AttributeContainingWithFindStrategy(attributeName, value));
     }
diff --git a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json
@@ -110,6 +110,10 @@
     "shopUrl": "http://demos.bellatrix.solutions/cart/",
     "accountUrl": "http://demos.bellatrix.solutions/account/"
   },
+  "openCvServiceSettings": {
+    "shouldGrayscale": true,
+    "defaultMatchThreshold": "0.9"
+  },
   "testPagesSettings": {
     "anchorLocalPage": "testpages/anchor/anchor.html",
     "buttonLocalPage": "testpages/button/button.html",
diff --git a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json
@@ -110,6 +110,10 @@
     "shopUrl": "http://demos.bellatrix.solutions/cart/",
     "accountUrl": "http://demos.bellatrix.solutions/account/"
   },
+  "openCvServiceSettings": {
+    "shouldGrayscale": true,
+    "defaultMatchThreshold": "0.9"
+  },
   "testPagesSettings": {
     "anchorLocalPage": "testpages/anchor/anchor.html",
     "buttonLocalPage": "testpages/button/button.html",
diff --git a/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java b/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java
@@ -22,7 +22,7 @@ public void beforeEach() throws Exception {
 
     @Test
     public void actionPerformed_when_convertBase64ToImage_and_clickImage() {
-        var falcon9Image = app().create().byImage(Anchor.class, EncodedImageDemo.FALCON_9);
+        var falcon9Image = app().create().byImage(Anchor.class, EncodedImageDemo.FALCON_9, false, 0.99);
 
         app().navigate().to("http://demos.bellatrix.solutions/");
         falcon9Image.click();

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ public Point getLocation() {`
`32`	`32`
`33`	`33`	`var encodedImage = findStrategy.getEncodedImage();`
`34`	`34`
`35`		`- var location = OpenCvService.getLocation(encodedImage, true);`
	`35`	`+ var location = OpenCvService.getLocation(encodedImage);`
`36`	`36`	`return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset());`
`37`	`37`	`}`
`38`	`38`
Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ public Point getLocation() {`
`30`	`30`
`31`	`31`	`var encodedImage = findStrategy.getEncodedImage();`
`32`	`32`
`33`		`- var location = OpenCvService.getLocation(encodedImage, true);`
	`33`	`+ var location = OpenCvService.getLocation(encodedImage);`
`34`	`34`	`return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset());`
`35`	`35`	`}`
`36`	`36`