Skip to content

feat/CUS-11616-Added class verify the text is present on screen in if condition #367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akhil-testsigma merged 1 commit into from
Mar 30, 2026
Merged

feat/CUS-11616-Added class verify the text is present on screen in if condition #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions image_based_actions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,18 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.testsigma.addons</groupId>
<artifactId>image_based_actions</artifactId>
<version>1.0.14</version>
<version>1.0.16</version>
<packaging>jar</packaging>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<testsigma.sdk.version>1.2.6_cloud</testsigma.sdk.version>
<testsigma.sdk.version>1.2.24_cloud</testsigma.sdk.version>
<junit.jupiter.version>5.8.0-M1</junit.jupiter.version>
<testsigma.addon.maven.plugin>1.0.0</testsigma.addon.maven.plugin>
<maven.source.plugin.version>3.2.1</maven.source.plugin.version>
<lombok.version>1.18.20</lombok.version>
<lombok.version>1.18.30</lombok.version>

</properties>

Expand Down Expand Up @@ -48,13 +48,13 @@
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>4.14.1</version>
<version>4.33.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.appium/java-client -->
<dependency>
<groupId>io.appium</groupId>
<artifactId>java-client</artifactId>
<version>9.0.0</version>
<version>9.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand All @@ -64,7 +64,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.14.0</version>
<version>3.17.0</version>
</dependency>

</dependencies>
Expand All @@ -83,7 +83,7 @@
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>true</minimizeJar>
<minimizeJar>false</minimizeJar>
<filters>
<filter>
<artifact>*:*</artifact>
Expand Down
6 changes: 3 additions & 3 deletions image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,19 @@ protected Result execute() throws NoSuchElementException {
File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE);

// Switch to native app context
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP"));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context("NATIVE_APP");

String successMessage= performClick(ocr, testData1.getValue().toString(), null,
baseImageFile,testStepResult.getScreenshotUrl(), appiumDriver, logger);

setSuccessMessage(successMessage);

// Switch back to old context
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext);

return Result.SUCCESS;
} catch (Exception e) {
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext);
logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click on image: " + e.getMessage());
return Result.FAILED;
Expand Down
6 changes: 3 additions & 3 deletions ...ased_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ protected Result execute() throws NoSuchElementException {
File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE);

// Switch to native app context
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP"));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context("NATIVE_APP");

String successMessage = performClick(ocr, testData1.getValue().toString(),
Float.valueOf(testData2.getValue().toString()), baseImageFile, testStepResult.getScreenshotUrl(),
Expand All @@ -57,11 +57,11 @@ protected Result execute() throws NoSuchElementException {
setSuccessMessage(successMessage);

// Switch back to old context
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext);

return Result.SUCCESS;
} catch (Exception e) {
appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext));
((io.appium.java_client.remote.SupportsContextSwitching) appiumDriver).context(existingContext);
logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click on image: " + e.getMessage());
return Result.FAILED;
Expand Down
27 changes: 2 additions & 25 deletions image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,14 @@
package com.testsigma.addons.mobile_web.util;

import com.google.common.collect.ImmutableMap;
import io.appium.java_client.AppiumDriver;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.remote.DriverCommand;
import org.openqa.selenium.remote.Response;

import java.util.LinkedHashSet;
import java.util.Set;

public class ContextUtils {
public static Set<String> getContextHandles(AppiumDriver driver) {
Response response = driver.execute(DriverCommand.GET_CONTEXT_HANDLES, ImmutableMap.of());
Object value = response.getValue();
try {
//noinspection unchecked
java.util.List<String> returnedValues = (java.util.List<String>) value;
return new LinkedHashSet<>(returnedValues);
} catch (ClassCastException ex) {
throw new WebDriverException(
"Returned value cannot be converted to List<String>: " + value, ex);
}
return ((io.appium.java_client.remote.SupportsContextSwitching) driver).getContextHandles();
}

public static String getCurrentContext(AppiumDriver driver) {
Response response = driver.execute(DriverCommand.GET_CURRENT_CONTEXT_HANDLE, ImmutableMap.of());
Object value = response.getValue();
try {
//noinspection unchecked
return (String) value;
} catch (ClassCastException ex) {
throw new WebDriverException(
"Returned value cannot be converted to List<String>: " + value, ex);
}
return ((io.appium.java_client.remote.SupportsContextSwitching) driver).getContext();
}
}
59 changes: 29 additions & 30 deletions image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import org.openqa.selenium.chromium.ChromiumDriver;
import org.openqa.selenium.edge.EdgeDriver;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.safari.SafariDriver;;
import org.openqa.selenium.safari.SafariDriver;

import javax.imageio.ImageIO;
import java.awt.*;
Expand All @@ -22,10 +22,7 @@
import java.util.List;

@Data
@Action(actionText = "Click on text testdata on system/desktop window",
description = "Click on the text using the text coordinates in system/desktop window",
applicationType = ApplicationType.WEB,
useCustomScreenshot = true)
@Action(actionText = "Click on text testdata on system/desktop window", description = "Click on the text using the text coordinates in system/desktop window", applicationType = ApplicationType.WEB, useCustomScreenshot = true)

public class ClickOnTextSystemOrDesktop extends WebAction {
@OCR
Expand All @@ -40,9 +37,10 @@ public class ClickOnTextSystemOrDesktop extends WebAction {
@Override
protected Result execute() {
Result result = Result.SUCCESS;
try{
try {
if (isCloudExecution()) {
setErrorMessage("This action (Click on text testdata on system/desktop window) is not supported in cloud execution environments.");
setErrorMessage(
"This action (Click on text testdata on system/desktop window) is not supported in cloud execution environments.");
result = Result.FAILED;
return result;
}
Expand All @@ -56,12 +54,12 @@ protected Result execute() {

// Provide the destination details to copy the screenshot
String tempDir = System.getProperty("java.io.tmpdir");
String filename = "screenshot"+System.currentTimeMillis()+".jpg";
String filename = "screenshot" + System.currentTimeMillis() + ".jpg";
String path = tempDir + filename;
logger.info("Path: " + path);

// To copy source image in to destination path
ImageIO.write(tmp, "jpg",new File(path));
ImageIO.write(tmp, "jpg", new File(path));
int width = tmp.getWidth();
int height = tmp.getHeight();
logger.info("Width of image: " + width);
Expand All @@ -74,49 +72,50 @@ protected Result execute() {
List<OCRTextPoint> textPoints = ocr.extractTextFromImage(ocrImage);
printAllCoordinates(textPoints);
OCRTextPoint textPoint = getTextPointFromText(textPoints);
if(textPoint == null) {
if (textPoint == null) {
result = Result.FAILED;
setErrorMessage("Given text is not found");

} else {
logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() +
logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() +
", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2());
clickOnCoordinates(textPoint, width, height);
tmp = robot.createScreenCapture(screenSize);
filename = "screenshot"+System.currentTimeMillis()+".jpg";
filename = "screenshot" + System.currentTimeMillis() + ".jpg";
path = tempDir + filename;
ImageIO.write(tmp, "jpg",new File(path));
ImageIO.write(tmp, "jpg", new File(path));
baseImageFile = new File(path);
String url = testStepResult.getScreenshotUrl();
ocr.uploadFile(url, baseImageFile);
setSuccessMessage("Click operation performed on the text " +
" Text coordinates :" + "x-" + (int)xrelative + ", y-" + (int)yrelative);
" Text coordinates :" + "x-" + (int) xrelative + ", y-" + (int) yrelative);
}
} catch (Exception e){
logger.info("Exception: "+ Arrays.toString(e.getStackTrace()));
} catch (Exception e) {
logger.info("Exception: " + Arrays.toString(e.getStackTrace()));
setErrorMessage("Exception occurred while searching for the given text");
result = Result.FAILED;
}


return result;
}

private OCRTextPoint getTextPointFromText(List<OCRTextPoint> textPoints) {
if(textPoints == null) {
if (textPoints == null) {
return null;
}
for(OCRTextPoint textPoint: textPoints) {
if(text.getValue().equals(textPoint.getText())) {
for (OCRTextPoint textPoint : textPoints) {
if (text.getValue().equals(textPoint.getText())) {
return textPoint;

}
}
return null;
return null;
}

private void printAllCoordinates(List<OCRTextPoint> textPoints) {
for(OCRTextPoint textPoint: textPoints) {
logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n");
for (OCRTextPoint textPoint : textPoints) {
logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1()
+ ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() + "\n\n\n\n");
Comment on lines 115 to +118

@coderabbitai coderabbitai Bot Mar 27, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Add a null/empty guard before iterating OCR points.

printAllCoordinates can throw when OCR returns null; that failure is avoidable and currently routes through generic exception handling.

Suggested fix 
 private void printAllCoordinates(List<OCRTextPoint> textPoints) {
+    if (textPoints == null || textPoints.isEmpty()) {
+        logger.info("No OCR text points found");
+        return;
+    }
     for (OCRTextPoint textPoint : textPoints) {
         logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1()
                 + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() + "\n\n\n\n");
     }
 }
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In
`@image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnTextSystemOrDesktop.java`
around lines 115 - 118, The printAllCoordinates method should guard against a
null or empty textPoints list to prevent NPEs when OCR returns null; update
printAllCoordinates to check if textPoints is null or empty and return early
(optionally log a debug/info message) instead of iterating, and also defensively
handle any null OCRTextPoint entries (check textPoint != null before calling
getText/getX1/getY1/getX2/getY2) to avoid further null-pointer issues.

}
}

Expand All @@ -142,12 +141,11 @@ public void clickOnCoordinates(OCRTextPoint textPoint, int imageWidth, int image
long browserHeight = (Long) js.executeScript("return window.innerHeight;");
long browserWidth = (Long) js.executeScript("return window.innerWidth;");

xrelative = ((double)x / (double)browserWidth) * (double)imageWidth;
yrelative = ((double)y / (double)browserHeight) * (double)imageHeight;

xrelative = ((double) x / (double) browserWidth) * (double) imageWidth;
yrelative = ((double) y / (double) browserHeight) * (double) imageHeight;

logger.info("X relative: " + (int)xrelative + "\n");
logger.info("Y relative: " + (int)yrelative + "\n");
logger.info("X relative: " + (int) xrelative + "\n");
logger.info("Y relative: " + (int) yrelative + "\n");

robot.mouseMove((int) xrelative, (int) yrelative);
robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
Expand All @@ -157,10 +155,11 @@ public void clickOnCoordinates(OCRTextPoint textPoint, int imageWidth, int image
private boolean isCloudExecution() {
try {
// If the driver is an instance of any driver than it is local
if (driver instanceof ChromiumDriver || driver instanceof EdgeDriver || driver instanceof FirefoxDriver || driver instanceof SafariDriver) {
if (driver instanceof ChromiumDriver || driver instanceof EdgeDriver || driver instanceof FirefoxDriver
|| driver instanceof SafariDriver) {
return false;
} else {
//It's a cloud environment
// It's a cloud environment
return true;
}
} catch (Exception e) {
Expand Down
1 change: 1 addition & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ protected Result execute() throws NoSuchElementException {
logger.info("Height of image: " + height);

File baseImageFile = new File(path);
logger.info("Base image file: " + baseImageFile.getAbsolutePath());
String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
Expand Down
2 changes: 2 additions & 0 deletions ...e_based_actions/src/main/java/com/testsigma/addons/windows/ClickOnImageWithThreshold.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ protected Result execute() throws NoSuchElementException {
logger.info("Height of image: " + height);

File baseImageFile = new File(path);

logger.info("Base image file: " + baseImageFile.getAbsolutePath());
String url = testStepResult.getScreenshotUrl();
ocr.uploadFile(url, baseImageFile);
logger.info("url: "+ testStepResult.getScreenshotUrl());
Expand Down
Loading
Loading