Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat[TE-15256]: Image based Actions addon #41

Merged
merged 2 commits into from
Feb 23, 2024
Merged

Feat[TE-15256]: Image based Actions addon #41

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc7e5dc
Feat[TE-15256]: Image based Actions addon
SunilGembali Feb 5, 2024
902500d
Feat[TE-15256]: Added android nlp's and corrected code mistakes
SunilGembali Feb 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions image_based_actions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
<?xml version="1.0" encoding="UTF-8"?>
<project
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.testsigma.addons</groupId>
<artifactId>image_based_actions</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<testsigma.sdk.version>1.2.6_cloud</testsigma.sdk.version>
<junit.jupiter.version>5.8.0-M1</junit.jupiter.version>
<testsigma.addon.maven.plugin>1.0.0</testsigma.addon.maven.plugin>
<maven.source.plugin.version>3.2.1</maven.source.plugin.version>
<lombok.version>1.18.20</lombok.version>

</properties>

<dependencies>
<dependency>
<groupId>com.testsigma</groupId>
<artifactId>testsigma-java-sdk</artifactId>
<version>${testsigma.sdk.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.14.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.seleniumhq.selenium/selenium-java -->
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>4.14.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.appium/java-client -->
<dependency>
<groupId>io.appium</groupId>
<artifactId>java-client</artifactId>
<version>9.0.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.14.0</version>
</dependency>

</dependencies>
<build>
<finalName>image_based_actions</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven.source.plugin.version}</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
89 changes: 89 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package com.testsigma.addons.web;

import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.FindImageResponse;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WebAction;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Actions;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;

@Data
@Action(actionText = "Click on image image-url",
description = "Click on give image",
applicationType = ApplicationType.WEB,
useCustomScreenshot = true)
public class ClickOnImage extends WebAction {
@TestData(reference = "image-url")
private com.testsigma.sdk.TestData testData1;

@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
TakesScreenshot scrShot = ((TakesScreenshot) this.driver);
File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int width = bufferedImage.getWidth();
int height = bufferedImage.getHeight();
logger.info("Width of image: " + width);
logger.info("Height of image: " + height);

long innerHeight = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerHeight;");
long innerWidth = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerWidth;");
int windowHeight = (int) innerHeight;
int windowWidth = (int) innerWidth;
logger.info("inner Width of window: " + windowWidth);
String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString());
if(response.getIsFound()) {
int clickLocationX = (response.getX1() + response.getX2()) / 2;
int clickLocationY = (response.getY1() + response.getY2()) / 2;

double xRelative = (double) clickLocationX / width;
double yRelative = (double) clickLocationY / height;

int xCoordinate = (int) (xRelative * windowWidth);
int yCoordinate = (int) (yRelative * windowHeight);
logger.info("Click Location X: " + xCoordinate);
logger.info("Click Location Y: " + yCoordinate);

Actions actions = new Actions(driver);
actions.moveToLocation(xCoordinate, yCoordinate).click().build().perform();
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}

}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
93 changes: 93 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImageOccurrenceBased.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package com.testsigma.addons.web;

import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.FindImageResponse;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WebAction;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Actions;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;

@Data
@Action(actionText = "Click on image image-url, occurrence position found-at-position",
description = "Click on give image at the given position",
applicationType = ApplicationType.WEB,
useCustomScreenshot = true)
public class ClickOnImageOccurrenceBased extends WebAction {
@TestData(reference = "image-url")
private com.testsigma.sdk.TestData testData1;

@TestData(reference = "found-at-position")
private com.testsigma.sdk.TestData testData2;

@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
TakesScreenshot scrShot = ((TakesScreenshot) this.driver);
File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int width = bufferedImage.getWidth();
int height = bufferedImage.getHeight();
logger.info("Width of image: " + width);
logger.info("Height of image: " + height);

long innerHeight = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerHeight;");
long innerWidth = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerWidth;");
int windowHeight = (int) innerHeight;
int windowWidth = (int) innerWidth;
logger.info("inner Width of window: " + windowWidth);
logger.info("inner Height of window: " + windowHeight);

String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Integer.parseInt(testData2.getValue().toString()));
if(response.getIsFound()) {
int clickLocationX = (response.getX1() + response.getX2()) / 2;
int clickLocationY = (response.getY1() + response.getY2()) / 2;

double xRelative = (double) clickLocationX / width;
double yRelative = (double) clickLocationY / height;

int xCoordinate = (int) (xRelative * windowWidth);
int yCoordinate = (int) (yRelative * windowHeight);
logger.info("Click Location X: " + xCoordinate);
logger.info("Click Location Y: " + yCoordinate);

Actions actions = new Actions(driver);
actions.moveToLocation(xCoordinate, yCoordinate).click().build().perform();
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}
}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
92 changes: 92 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImageWithThreshold.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package com.testsigma.addons.web;

import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.FindImageResponse;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WebAction;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Actions;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;

@Data
@Action(actionText = "Click on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match)",
description = "Click on given image with threshold",
applicationType = ApplicationType.WEB)
public class ClickOnImageWithThreshold extends WebAction {
@TestData(reference = "image-file")
private com.testsigma.sdk.TestData testData1;
@TestData(reference = "threshold-value")
private com.testsigma.sdk.TestData testData2;


@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
TakesScreenshot scrShot = ((TakesScreenshot) this.driver);
File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int width = bufferedImage.getWidth();
int height = bufferedImage.getHeight();
logger.info("Width of image: " + width);
logger.info("Height of image: " + height);

long innerHeight = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerHeight;");
long innerWidth = (Long) ((JavascriptExecutor) driver).executeScript("return window.innerWidth;");
int windowHeight = (int) innerHeight;
int windowWidth = (int) innerWidth;
logger.info("inner Width of window: " + windowWidth);
logger.info("inner Height of window: " + windowHeight);

String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString()));
if(response.getIsFound()) {
int clickLocationX = (response.getX1() + response.getX2()) / 2;
int clickLocationY = (response.getY1() + response.getY2()) / 2;

double xRelative = (double) clickLocationX / width;
double yRelative = (double) clickLocationY / height;

int xCoordinate = (int) (xRelative * windowWidth);
int yCoordinate = (int) (yRelative * windowHeight);
logger.info("Click Location X: " + xCoordinate);
logger.info("Click Location Y: " + yCoordinate);

Actions actions = new Actions(driver);
actions.moveToLocation(xCoordinate, yCoordinate).click().build().perform();
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}
}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
Loading