Feat[TE-15256]: Image based Actions addon (#41)

* Feat[TE-15256]: Image based Actions addon * Feat[TE-15256]: Added android nlp's and corrected code mistakes
testsigmahq · Feb 23, 2024 · 56f9606 · 56f9606
1 parent a17effa
commit 56f9606
Show file tree

Hide file tree

Showing 30 changed files with 2,924 additions and 0 deletions.
diff --git a/image_based_actions/pom.xml b/image_based_actions/pom.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project
+    xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.testsigma.addons</groupId>
+    <artifactId>image_based_actions</artifactId>
+    <version>1.0.3</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <testsigma.sdk.version>1.2.6_cloud</testsigma.sdk.version>
+        <junit.jupiter.version>5.8.0-M1</junit.jupiter.version>
+        <testsigma.addon.maven.plugin>1.0.0</testsigma.addon.maven.plugin>
+        <maven.source.plugin.version>3.2.1</maven.source.plugin.version>
+        <lombok.version>1.18.20</lombok.version>
+
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.testsigma</groupId>
+            <artifactId>testsigma-java-sdk</artifactId>
+            <version>${testsigma.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>${lombok.version}</version>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.14.3</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.seleniumhq.selenium/selenium-java -->
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>4.14.1</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/io.appium/java-client -->
+        <dependency>
+            <groupId>io.appium</groupId>
+            <artifactId>java-client</artifactId>
+            <version>9.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>2.13.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.14.0</version>
+        </dependency>
+
+    </dependencies>
+    <build>
+        <finalName>image_based_actions</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>${maven.source.plugin.version}</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/image_based_actions/src/main/java/com/testsigma/addons/android/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/android/ClickOnImage.java
@@ -0,0 +1,111 @@
+package com.testsigma.addons.android;
+
+import com.testsigma.sdk.AndroidAction;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import io.appium.java_client.android.AndroidDriver;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.openqa.selenium.Dimension;
+import org.openqa.selenium.NoSuchElementException;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.TakesScreenshot;
+import org.openqa.selenium.interactions.Pause;
+import org.openqa.selenium.interactions.PointerInput;
+import org.openqa.selenium.interactions.Sequence;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.Arrays;
+
+import static java.time.Duration.ofMillis;
+import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH;
+import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT;
+import static org.openqa.selenium.interactions.PointerInput.Origin.viewport;
+
+@Data
+@Action(actionText = "Tap on image image-url",
+        description = "Tap on give image",
+        applicationType = ApplicationType.ANDROID)
+public class ClickOnImage extends AndroidAction {
+    @TestData(reference = "image-url")
+    private com.testsigma.sdk.TestData testData1;
+
+    @OCR
+    private com.testsigma.sdk.OCR ocr;
+
+    @TestStepResult
+    private com.testsigma.sdk.TestStepResult testStepResult;
+
+    @Override
+    protected Result execute() throws NoSuchElementException {
+        Result result = Result.SUCCESS;
+        try {
+            AndroidDriver androidDriver = (AndroidDriver) this.driver;
+            File baseImageFile = ((TakesScreenshot)androidDriver).getScreenshotAs(OutputType.FILE);
+            BufferedImage bufferedImage = ImageIO.read(baseImageFile);
+            int imageWidth = bufferedImage.getWidth();
+            int imageHeight = bufferedImage.getHeight();
+            logger.info("Width of image: " + imageWidth);
+            logger.info("Height of image: " + imageHeight);
+            Dimension dimension = androidDriver.manage().window().getSize();
+            int screenWidth = dimension.width;
+            int screenHeight = dimension.height;
+            logger.info("Screen width: "+screenWidth);
+            logger.info("Screen height: "+screenHeight);
+            String url = testStepResult.getScreenshotUrl();
+            logger.info("Amazon s3 url in which we are storing base image"+url);
+            ocr.uploadFile(url, baseImageFile);
+            FindImageResponse response = ocr.findImage(testData1.getValue().toString());
+            if(response.getIsFound()) {
+                logger.info("Image location found");
+                logger.info("Image Found :" + response.getIsFound() +
+                        "    Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
+                logger.info("Performing click..");
+                int x = (response.getX1() + response.getX2()) / 2;
+                int y = (response.getY1() + response.getY2()) / 2;
+                logger.info("Screen shot based click locations: x="+x+"y="+y);
+                double xRelative = ((double) x / imageWidth);
+                double yRelative = ((double) y / imageHeight);
+                logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative);
+                int clickLocationX;
+                int clickLocationY;
+                if (Math.abs(imageWidth-screenWidth) > 5) {
+                    clickLocationX = (int) (xRelative * screenWidth);
+                    clickLocationY = (int) (yRelative * screenHeight);
+                } else {
+                    clickLocationX = x;
+                    clickLocationY = y;
+                }
+                logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY);
+                PointerInput FINGER = new PointerInput(TOUCH, "finger");
+                Sequence tap = new Sequence(FINGER, 1)
+                        .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY))
+                        .addAction(FINGER.createPointerDown(LEFT.asArg()))
+                        .addAction(new Pause(FINGER, ofMillis(2)))
+                        .addAction(FINGER.createPointerUp(LEFT.asArg()));
+                androidDriver.perform(Arrays.asList(tap));
+                logger.info("CLick performed");
+                setSuccessMessage("Image Found :" + response.getIsFound() +
+                        "    Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
+                Thread.sleep(1000);
+            } else {
+                setErrorMessage("Unable to fetch the coordinates");
+                result = Result.FAILED;
+            }
+
+        }
+        catch (Exception e){
+            logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
+            setErrorMessage("Exception occurred while performing click action");
+            result = Result.FAILED;
+        }
+        return result;
+    }
+}
diff --git a/...based_actions/src/main/java/com/testsigma/addons/android/ClickOnImageOccurrenceBased.java b/...based_actions/src/main/java/com/testsigma/addons/android/ClickOnImageOccurrenceBased.java
@@ -0,0 +1,113 @@
+package com.testsigma.addons.android;
+
+import com.testsigma.sdk.AndroidAction;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import io.appium.java_client.android.AndroidDriver;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.openqa.selenium.Dimension;
+import org.openqa.selenium.NoSuchElementException;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.TakesScreenshot;
+import org.openqa.selenium.interactions.Pause;
+import org.openqa.selenium.interactions.PointerInput;
+import org.openqa.selenium.interactions.Sequence;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.Arrays;
+
+import static java.time.Duration.ofMillis;
+import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH;
+import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT;
+import static org.openqa.selenium.interactions.PointerInput.Origin.viewport;
+
+@Data
+@Action(actionText = "Tap on image image-url, occurrence position found-at-position",
+        description = "Tap on give image at the given position",
+        applicationType = ApplicationType.ANDROID)
+public class ClickOnImageOccurrenceBased extends AndroidAction {
+    @TestData(reference = "image-url")
+    private com.testsigma.sdk.TestData testData1;
+
+    @TestData(reference = "found-at-position")
+    private com.testsigma.sdk.TestData testData2;
+
+    @OCR
+    private com.testsigma.sdk.OCR ocr;
+
+    @TestStepResult
+    private com.testsigma.sdk.TestStepResult testStepResult;
+
+    @Override
+    protected Result execute() throws NoSuchElementException {
+        Result result = Result.SUCCESS;
+        try {
+            AndroidDriver androidDriver = (AndroidDriver) this.driver;
+            File baseImageFile = ((TakesScreenshot)androidDriver).getScreenshotAs(OutputType.FILE);
+            BufferedImage bufferedImage = ImageIO.read(baseImageFile);
+            int imageWidth = bufferedImage.getWidth();
+            int imageHeight = bufferedImage.getHeight();
+            logger.info("Width of image: " + imageWidth);
+            logger.info("Height of image: " + imageHeight);
+            Dimension dimension = androidDriver.manage().window().getSize();
+            int screenWidth = dimension.width;
+            int screenHeight = dimension.height;
+            logger.info("Screen width: "+screenWidth);
+            logger.info("Screen height: "+screenHeight);
+            String url = testStepResult.getScreenshotUrl();
+            logger.info("Amazon s3 url in which we are storing base image"+url);
+            ocr.uploadFile(url, baseImageFile);
+            FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Integer.parseInt(testData2.getValue().toString()));
+            if(response.getIsFound()) {
+                logger.info("Image location found");
+                logger.info("Image Found :" + response.getIsFound() +
+                        "    Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
+                logger.info("Performing click..");
+                int x = (response.getX1() + response.getX2()) / 2;
+                int y = (response.getY1() + response.getY2()) / 2;
+                logger.info("Screen shot based click locations: x="+x+"y="+y);
+                double xRelative = ((double) x / imageWidth);
+                double yRelative = ((double) y / imageHeight);
+                logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative);
+                int clickLocationX;
+                int clickLocationY;
+                if(Math.abs(imageWidth-screenWidth) > 5){
+                    clickLocationX = (int) (xRelative * screenWidth);
+                    clickLocationY = (int) (yRelative * screenHeight);
+                } else {
+                    clickLocationX = x;
+                    clickLocationY = y;
+                }
+                logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY);
+                PointerInput FINGER = new PointerInput(TOUCH, "finger");
+                Sequence tap = new Sequence(FINGER, 1)
+                        .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY))
+                        .addAction(FINGER.createPointerDown(LEFT.asArg()))
+                        .addAction(new Pause(FINGER, ofMillis(2)))
+                        .addAction(FINGER.createPointerUp(LEFT.asArg()));
+                androidDriver.perform(Arrays.asList(tap));
+                logger.info("CLick performed");
+                setSuccessMessage("Image Found :" + response.getIsFound() +
+                        "    Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
+                Thread.sleep(1000);
+            } else {
+                setErrorMessage("Unable to fetch the coordinates");
+                result = Result.FAILED;
+            }
+        }
+        catch (Exception e){
+            logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
+            setErrorMessage("Exception occurred while performing click action");
+            result = Result.FAILED;
+        }
+        return result;
+    }
+}