From 516a8f296f5b1b1f78766f6b84557581cf924652 Mon Sep 17 00:00:00 2001 From: Akhil Appini Date: Fri, 27 Mar 2026 20:50:11 +0530 Subject: [PATCH] Added class verify the text is present on screen in if condition --- image_based_actions/pom.xml | 14 +-- .../addons/mobile_web/ClickOnImage.java | 6 +- .../mobile_web/ClickOnImageWithThreshold.java | 6 +- .../addons/mobile_web/util/ContextUtils.java | 27 +---- .../web/ClickOnTextSystemOrDesktop.java | 59 +++++----- .../addons/windows/ClickOnImage.java | 1 + .../windows/ClickOnImageWithThreshold.java | 2 + .../windows/VerifyTextOnImageIfCondition.java | 111 ++++++++++++++++++ 8 files changed, 158 insertions(+), 68 deletions(-) create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/VerifyTextOnImageIfCondition.java diff --git a/image_based_actions/pom.xml b/image_based_actions/pom.xml index 398a6685..ef5a616d 100644 --- a/image_based_actions/pom.xml +++ b/image_based_actions/pom.xml @@ -6,18 +6,18 @@ 4.0.0 com.testsigma.addons image_based_actions - 1.0.14 + 1.0.16 jar UTF-8 11 11 - 1.2.6_cloud + 1.2.24_cloud 5.8.0-M1 1.0.0 3.2.1 - 1.18.20 + 1.18.30 @@ -48,13 +48,13 @@ org.seleniumhq.selenium selenium-java - 4.14.1 + 4.33.0 io.appium java-client - 9.0.0 + 9.4.0 com.fasterxml.jackson.core @@ -64,7 +64,7 @@ org.apache.commons commons-lang3 - 3.14.0 + 3.17.0 @@ -83,7 +83,7 @@ shade - true + false *:* diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java index d6542c03..9a77cb3a 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java @@ -44,7 +44,7 @@ protected Result execute() throws NoSuchElementException { File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE); // Switch to native app context - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP")); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context("NATIVE_APP"); String successMessage= performClick(ocr, testData1.getValue().toString(), null, baseImageFile,testStepResult.getScreenshotUrl(), appiumDriver, logger); @@ -52,11 +52,11 @@ protected Result execute() throws NoSuchElementException { setSuccessMessage(successMessage); // Switch back to old context - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext); return Result.SUCCESS; } catch (Exception e) { - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext); logger.info("Exception: " + ExceptionUtils.getStackTrace(e)); setErrorMessage("Exception occurred while performing click on image: " + e.getMessage()); return Result.FAILED; diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java index a9be241f..c32fee3a 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java @@ -48,7 +48,7 @@ protected Result execute() throws NoSuchElementException { File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE); // Switch to native app context - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP")); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context("NATIVE_APP"); String successMessage = performClick(ocr, testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString()), baseImageFile, testStepResult.getScreenshotUrl(), @@ -57,11 +57,11 @@ protected Result execute() throws NoSuchElementException { setSuccessMessage(successMessage); // Switch back to old context - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext); return Result.SUCCESS; } catch (Exception e) { - appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + ((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext); logger.info("Exception: " + ExceptionUtils.getStackTrace(e)); setErrorMessage("Exception occurred while performing click on image: " + e.getMessage()); return Result.FAILED; diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java index 8a5db32b..a7bfeab7 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java @@ -1,37 +1,14 @@ package com.testsigma.addons.mobile_web.util; -import com.google.common.collect.ImmutableMap; import io.appium.java_client.AppiumDriver; -import org.openqa.selenium.WebDriverException; -import org.openqa.selenium.remote.DriverCommand; -import org.openqa.selenium.remote.Response; - -import java.util.LinkedHashSet; import java.util.Set; public class ContextUtils { public static Set getContextHandles(AppiumDriver driver) { - Response response = driver.execute(DriverCommand.GET_CONTEXT_HANDLES, ImmutableMap.of()); - Object value = response.getValue(); - try { - //noinspection unchecked - java.util.List returnedValues = (java.util.List) value; - return new LinkedHashSet<>(returnedValues); - } catch (ClassCastException ex) { - throw new WebDriverException( - "Returned value cannot be converted to List: " + value, ex); - } + return ((io.appium.java_client.remote.SupportsContextSwitching) driver).getContextHandles(); } public static String getCurrentContext(AppiumDriver driver) { - Response response = driver.execute(DriverCommand.GET_CURRENT_CONTEXT_HANDLE, ImmutableMap.of()); - Object value = response.getValue(); - try { - //noinspection unchecked - return (String) value; - } catch (ClassCastException ex) { - throw new WebDriverException( - "Returned value cannot be converted to List: " + value, ex); - } + return ((io.appium.java_client.remote.SupportsContextSwitching) driver).getContext(); } } diff --git a/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java b/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java index c9e4097e..3aee30ba 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java @@ -11,7 +11,7 @@ import org.openqa.selenium.chromium.ChromiumDriver; import org.openqa.selenium.edge.EdgeDriver; import org.openqa.selenium.firefox.FirefoxDriver; -import org.openqa.selenium.safari.SafariDriver;; +import org.openqa.selenium.safari.SafariDriver; import javax.imageio.ImageIO; import java.awt.*; @@ -22,10 +22,7 @@ import java.util.List; @Data -@Action(actionText = "Click on text testdata on system/desktop window", - description = "Click on the text using the text coordinates in system/desktop window", - applicationType = ApplicationType.WEB, - useCustomScreenshot = true) +@Action(actionText = "Click on text testdata on system/desktop window", description = "Click on the text using the text coordinates in system/desktop window", applicationType = ApplicationType.WEB, useCustomScreenshot = true) public class ClickOnTextSystemOrDesktop extends WebAction { @OCR @@ -40,9 +37,10 @@ public class ClickOnTextSystemOrDesktop extends WebAction { @Override protected Result execute() { Result result = Result.SUCCESS; - try{ + try { if (isCloudExecution()) { - setErrorMessage("This action (Click on text testdata on system/desktop window) is not supported in cloud execution environments."); + setErrorMessage( + "This action (Click on text testdata on system/desktop window) is not supported in cloud execution environments."); result = Result.FAILED; return result; } @@ -56,12 +54,12 @@ protected Result execute() { // Provide the destination details to copy the screenshot String tempDir = System.getProperty("java.io.tmpdir"); - String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String filename = "screenshot" + System.currentTimeMillis() + ".jpg"; String path = tempDir + filename; logger.info("Path: " + path); // To copy source image in to destination path - ImageIO.write(tmp, "jpg",new File(path)); + ImageIO.write(tmp, "jpg", new File(path)); int width = tmp.getWidth(); int height = tmp.getHeight(); logger.info("Width of image: " + width); @@ -74,49 +72,50 @@ protected Result execute() { List textPoints = ocr.extractTextFromImage(ocrImage); printAllCoordinates(textPoints); OCRTextPoint textPoint = getTextPointFromText(textPoints); - if(textPoint == null) { + if (textPoint == null) { result = Result.FAILED; setErrorMessage("Given text is not found"); } else { - logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); clickOnCoordinates(textPoint, width, height); tmp = robot.createScreenCapture(screenSize); - filename = "screenshot"+System.currentTimeMillis()+".jpg"; + filename = "screenshot" + System.currentTimeMillis() + ".jpg"; path = tempDir + filename; - ImageIO.write(tmp, "jpg",new File(path)); + ImageIO.write(tmp, "jpg", new File(path)); baseImageFile = new File(path); String url = testStepResult.getScreenshotUrl(); ocr.uploadFile(url, baseImageFile); setSuccessMessage("Click operation performed on the text " + - " Text coordinates :" + "x-" + (int)xrelative + ", y-" + (int)yrelative); + " Text coordinates :" + "x-" + (int) xrelative + ", y-" + (int) yrelative); } - } catch (Exception e){ - logger.info("Exception: "+ Arrays.toString(e.getStackTrace())); + } catch (Exception e) { + logger.info("Exception: " + Arrays.toString(e.getStackTrace())); setErrorMessage("Exception occurred while searching for the given text"); result = Result.FAILED; } - return result; } + private OCRTextPoint getTextPointFromText(List textPoints) { - if(textPoints == null) { + if (textPoints == null) { return null; } - for(OCRTextPoint textPoint: textPoints) { - if(text.getValue().equals(textPoint.getText())) { + for (OCRTextPoint textPoint : textPoints) { + if (text.getValue().equals(textPoint.getText())) { return textPoint; } } - return null; + return null; } private void printAllCoordinates(List textPoints) { - for(OCRTextPoint textPoint: textPoints) { - logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + for (OCRTextPoint textPoint : textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() + "\n\n\n\n"); } } @@ -142,12 +141,11 @@ public void clickOnCoordinates(OCRTextPoint textPoint, int imageWidth, int image long browserHeight = (Long) js.executeScript("return window.innerHeight;"); long browserWidth = (Long) js.executeScript("return window.innerWidth;"); - xrelative = ((double)x / (double)browserWidth) * (double)imageWidth; - yrelative = ((double)y / (double)browserHeight) * (double)imageHeight; - + xrelative = ((double) x / (double) browserWidth) * (double) imageWidth; + yrelative = ((double) y / (double) browserHeight) * (double) imageHeight; - logger.info("X relative: " + (int)xrelative + "\n"); - logger.info("Y relative: " + (int)yrelative + "\n"); + logger.info("X relative: " + (int) xrelative + "\n"); + logger.info("Y relative: " + (int) yrelative + "\n"); robot.mouseMove((int) xrelative, (int) yrelative); robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); @@ -157,10 +155,11 @@ public void clickOnCoordinates(OCRTextPoint textPoint, int imageWidth, int image private boolean isCloudExecution() { try { // If the driver is an instance of any driver than it is local - if (driver instanceof ChromiumDriver || driver instanceof EdgeDriver || driver instanceof FirefoxDriver || driver instanceof SafariDriver) { + if (driver instanceof ChromiumDriver || driver instanceof EdgeDriver || driver instanceof FirefoxDriver + || driver instanceof SafariDriver) { return false; } else { - //It's a cloud environment + // It's a cloud environment return true; } } catch (Exception e) { diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImage.java index 04693d00..c0338692 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImage.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImage.java @@ -59,6 +59,7 @@ protected Result execute() throws NoSuchElementException { logger.info("Height of image: " + height); File baseImageFile = new File(path); + logger.info("Base image file: " + baseImageFile.getAbsolutePath()); String url = testStepResult.getScreenshotUrl(); logger.info("Amazon s3 url in which we are storing base image"+url); ocr.uploadFile(url, baseImageFile); diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageWithThreshold.java index dff1e84b..d4804b4e 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageWithThreshold.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageWithThreshold.java @@ -61,6 +61,8 @@ protected Result execute() throws NoSuchElementException { logger.info("Height of image: " + height); File baseImageFile = new File(path); + + logger.info("Base image file: " + baseImageFile.getAbsolutePath()); String url = testStepResult.getScreenshotUrl(); ocr.uploadFile(url, baseImageFile); logger.info("url: "+ testStepResult.getScreenshotUrl()); diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/VerifyTextOnImageIfCondition.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/VerifyTextOnImageIfCondition.java new file mode 100644 index 00000000..44b4a68e --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/VerifyTextOnImageIfCondition.java @@ -0,0 +1,111 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; +import java.util.List; + +@Data +@Action(actionText = "Verify that the text text-value is present on the screen", + description = "Verify whether the text is present or not", + actionType = StepActionType.IF_CONDITION, + applicationType = ApplicationType.WINDOWS, + useCustomScreenshot = true) + +public class VerifyTextOnImageIfCondition extends WindowsAction { + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestData(reference = "text-value") + private com.testsigma.sdk.TestData text; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() { + Result result = Result.SUCCESS; + String inputText = text.getValue().toString().replaceAll("\\s+", ""); + try{ + Robot robot = new Robot(); + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + OCRImage ocrImage = new OCRImage(); + ocrImage.setOcrImageFile(baseImageFile); + + List textPoints = ocr.extractTextFromImage(ocrImage); + printAllCoordinates(textPoints); + OCRTextPoint textPoint = getTextPointFromText(textPoints,inputText); + if(textPoint == null) { + result = Result.FAILED; + setErrorMessage("Given text is not found"); + + } else { + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); + tmp = robot.createScreenCapture(screenSize); + filename = "screenshot"+System.currentTimeMillis()+".jpg"; + path = tempDir + filename; + ImageIO.write(tmp, "jpg",new File(path)); + baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + setSuccessMessage("Text is found at " + + " Text coordinates :" + "x1-" + textPoint.getX1() + ", x2-" + textPoint.getX2() + ", y1-" + textPoint.getY1() + ", y2-" + textPoint.getY2()); + } + } catch (Exception e){ + logger.info("Exception: "+ Arrays.toString(e.getStackTrace())); + setErrorMessage("Exception occurred while searching for the given text"); + result = Result.FAILED; + } + + + return result; + } + private OCRTextPoint getTextPointFromText(List textPoints,String text) { + if(textPoints == null) { + return null; + } + for(OCRTextPoint textPoint: textPoints) { + if(text.contains(textPoint.getText())) { + return textPoint; + + } + } + return null; + } + + private void printAllCoordinates(List textPoints) { + for(OCRTextPoint textPoint: textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + } + } + +}