the first commit

README.MD

@@ -0,0 +1,32 @@
+# Captcha Solver for Ikabot
+Login Captcha Solver for [Ikabot](https://github.com/physics-sp/ikabot)
+## Source code
+ - The folder `collection` contains all the draggable icons
+ - `SolveCaptcha.py` contains a generic function to solve a login captcha
+ - `visualizeCaptchaSolver.py` can help you to see the process of detecting icons
+ - `testFunctionSolveCaptcha.py` shows you how to use `solveCaptcha()` function
+ - `ChopImg.py` contains a function that chops a Captcha image into several pieces
+ - `IkabotSimulation.py` is an example of use based on Ikabot
+## Getting Started
+**You will need to Install Tesseract**
+
+**1**. Install tesseract using windows installer available at:  [https://github.com/UB-Mannheim/tesseract/wiki](https://github.com/UB-Mannheim/tesseract/wiki)
+
+**2**. Note the tesseract path from the installation. Default installation path at the time of this edit was:  `C:\Users\USER\AppData\Local\Programs\Tesseract-OCR\tesseract.exe`. It may change so please check the installation path.
+
+**3**.  `pip install pytesseract`
+
+**4**. Set the tesseract path (`TESSERACT_PATH`) in the script
+## Contribution
+All images have been listed in the collection at the time of this edit.</br>
+If you want to add icons in the collection, you have to add the `.png` file in the folder `\collection`.
+
+## Screenshots
+**Visualizer :**
+
+![Screenshot Captcha Solver](https://i.ibb.co/ByT6fp6/Captcha-Solver.png)
+
+**Simulation :**
+
+![Screenshot Captcha Solver](https://i.ibb.co/nC5kNWg/2021-07-27-09-07-05-Window.png)
+

SolveCaptcha.py

@@ -0,0 +1,118 @@
+import pytesseract
+import cv2
+from PIL import Image
+import os
+import logging
+
+if os.name == 'nt':
+  TESSERACT_PATH = "C:/Users/soludev5/AppData/Local/Programs/Tesseract-OCR/tesseract.exe"  # <------ /!\ CHANGE THIS /!\
+  pytesseract.pytesseract.tesseract_cmd = TESSERACT_PATH
+
+NUMBER_OF_IMAGE_IN_CAPTCHA = 4
+
+def extractText(textImg):
+    """This function returns a string which represents the name of the image that the text in `textImg` is describing.
+    Parameters
+    ----------
+    textImg : PIL.Image
+        a picture that contains text, which can be OCR-ed by tesseract.
+        
+    Returns
+    -------
+    captchaImgTitle : str
+        a string representing the name of the image that `textImg` is describing.
+    """
+
+    new_img = Image.new("RGB", textImg.size, (0, 0, 0))
+    new_img.paste(textImg, mask=textImg.split()[3])  # 3 is the alpha channel
+
+    # Perform text extraction
+    data = pytesseract.image_to_string(new_img, lang='eng')
+
+    # Format
+    data.strip()
+    sentence = data[:-3]
+
+    logging.info(sentence)
+
+    # format string
+    captchaImgTitle = sentence.split('onto')[0].split('Drag the')[-1].strip()
+
+    return captchaImgTitle
+
+
+def solveCaptcha(textImg, captchaImg, collectionPath):
+    """This function return an integer in the range [0,3]. This integer represents the ordinal position of the image described textually in `textImg`, found in `collectionPath` within `captchaImg`.
+    Parameters
+    ----------
+    textImg : PIL.Image
+        a picture that contains text, which can be OCR-ed by tesseract.
+    captchaImg : numpy.ndarray
+        a picture that contains within itself 4 pictures. This function will search for the index of the image described in `textImg` within this image and return it.
+        
+    Returns
+    -------
+    resultReturn : int
+        an integer representing the index of the image described in `textImg` within `captchaImg`.
+    """
+    ##############
+    # Read Text  #
+    ##############
+
+    captchaImgTitle = extractText(textImg)
+
+    # print string
+    logging.info(captchaImgTitle)
+
+    ################################
+    # Search for img in collection #
+    ################################
+
+    imgName = captchaImgTitle.lower().replace(' ', '_') + ".png"
+    assert os.path.isfile(collectionPath + imgName), "Image not found"
+
+    logging.info("Image found in collection :D")
+
+    #########################
+    # Detect img in Captcha #
+    #########################
+
+    method = cv2.TM_SQDIFF_NORMED
+
+    # Read the images from the file
+    small_image = cv2.imread(collectionPath + imgName)
+    large_image = captchaImg
+
+    result = cv2.matchTemplate(small_image, large_image, method)
+
+    # We want the minimum squared difference
+    mn, _, mnLoc, _ = cv2.minMaxLoc(result)
+
+    # Extract the coordinates of our best match
+    MPx, MPy = mnLoc
+
+    # Get the size of the template. This is the same size as the match.
+    trows, tcols = small_image.shape[:2]
+
+    # Get the coordinates of the template center on large_image
+    centerPointx = MPx + int(tcols/2)
+    centerPointy = MPy + int(trows/2)
+
+    #################
+    # Return number #
+    #################
+
+    # Get the width of large_image
+    largeWidth = large_image.shape[1]
+    # Get the width of 1/N large_image
+    widthQuarter = largeWidth/NUMBER_OF_IMAGE_IN_CAPTCHA
+
+    # Check the location of the centerPointx
+    for i in range(0, NUMBER_OF_IMAGE_IN_CAPTCHA):
+        if centerPointx >= widthQuarter*i and centerPointx < widthQuarter*(i+1):
+            resultReturn = i
+            break
+
+    logging.info("img n°", resultReturn+1)
+
+    return resultReturn

tests/ChopImg.py

@@ -0,0 +1,15 @@
+def chopImg(captchaImg):
+    # Chop img into pieces
+    NUMBER_OF_CHOPS = 4
+    arrayImg = []
+    width, height = captchaImg.size
+    chopsize = int(width/NUMBER_OF_CHOPS)
+
+    for x0 in range(0, width, chopsize):
+        for y0 in range(0, height, chopsize):
+            box = (x0, y0,
+                   x0+chopsize if x0+chopsize < width else width - 1,
+                   y0+chopsize if y0+chopsize < height else height - 1)
+            arrayImg.append(captchaImg.crop(box))
+
+    return arrayImg

tests/IkabotSimulation.py

@@ -0,0 +1,87 @@
+from ChopImg import chopImg
+import cv2
+from PIL import Image
+import sys
+import os
+sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) )
+from SolveCaptcha import *
+
+COLLECTION_FOLDER_PATH = "collection/"
+
+
+def saveImg(textImg, captchaImg, response):
+    name = extractText(textImg)
+    # convert from openCV2 to PIL. Notice the COLOR_BGR2RGB which means that
+    # the color is converted from BGR to RGB
+    color_coverted = cv2.cvtColor(captchaImg, cv2.COLOR_BGR2RGB)
+    pil_image = Image.fromarray(color_coverted)
+
+    array = chopImg(pil_image)
+
+    path = os.path.join(COLLECTION_FOLDER_PATH, name.lower().replace(' ','_') + ".png")
+
+    img = array[int(response)-1]
+    img.save(path)
+
+    print("Image saved in collection")
+
+
+def sendToBot(textImg, captchaImg):
+    print("Sent to Telegram")
+    print("Please enter the number of the correct image (1, 2, 3 or 4)")
+    textImg.show()
+    cv2.imshow("captcha", captchaImg)
+    cv2.waitKey(0)
+    cv2.destroyAllWindows()
+
+
+def getUserResponse():
+    input1 = input()
+    return input1
+
+
+def main():
+    global solvedByTelegram
+    print("The interactive captcha has been presented")
+
+    while True:
+        TEXT_IMAGE_EXAMPLE_PATH = "tests/examples/txt2.png"  
+        CAPTCHA_IMAGE_EXAMPLE_PATH = "tests/examples/img2.png" 
+
+        textImg = Image.open(TEXT_IMAGE_EXAMPLE_PATH)
+        textImg.load()
+
+        captchaImg = cv2.imread(CAPTCHA_IMAGE_EXAMPLE_PATH)
+
+        print("Trying to solve the captcha...")
+        result = solveCaptcha(textImg, captchaImg, COLLECTION_FOLDER_PATH)
+        print(result)
+
+        if result == -1:
+            print("Can't solve the captcha. Please solve it via Telegram")
+            while True:
+                sendToBot(textImg, captchaImg)
+                response = getUserResponse()
+                if response == '':
+                    continue
+                solvedByTelegram = True
+                break
+        else:
+            solvedByTelegram = False
+
+        captcha_sent = {}
+        # captcha_sent = self.s.post('https://image-drop-challenge.gameforge.com/challenge/{}/en-GB'.format(challenge_id), json=data).json()
+        captcha_sent['status'] = "solved"
+        if captcha_sent['status'] == 'solved':
+            captchaSolved = True
+            if captchaSolved:
+                print("Captcha solved")
+                if solvedByTelegram:
+                    saveImg(textImg, captchaImg, response)
+                break
+            else:
+                continue
+
+
+if __name__ == '__main__':
+    main()

tests/testFunctionSolveCaptcha.py

@@ -0,0 +1,19 @@
+import cv2
+from PIL import Image
+import sys
+import os
+sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) )
+from SolveCaptcha import solveCaptcha
+
+TEXT_IMAGE_PATH = "tests/examples/txt1.png"
+CAPTCHA_IMAGE_PATH = "tests/examples/img1.png"
+COLLECTION_FOLDER_PATH = "collection/"
+
+textImg = Image.open(TEXT_IMAGE_PATH)
+textImg.load()
+
+captchaImg = cv2.imread(CAPTCHA_IMAGE_PATH)
+
+result = solveCaptcha(textImg, captchaImg, COLLECTION_FOLDER_PATH)
+
+print(result)

tests/visualizeCaptchaSolver.py

@@ -0,0 +1,111 @@
+import pytesseract
+import cv2
+from PIL import Image
+import os
+
+TESSERACT_PATH = "C:/Users/soludev5/AppData/Local/Programs/Tesseract-OCR/tesseract.exe"
+COLLECTION_FOLDER_PATH = "collection/"
+
+TEXT_IMAGE_PATH = "tests/examples/txt2.png"
+CAPTCHA_IMAGE_PATH = "tests/examples/img2.png"
+
+##############
+# Read Text  #
+##############
+
+pytesseract.pytesseract.tesseract_cmd = TESSERACT_PATH
+
+png = Image.open(TEXT_IMAGE_PATH)
+png.load()  # required for png.split()
+
+new_img = Image.new("RGB", png.size, (0, 0, 0))
+new_img.paste(png, mask=png.split()[3])  # 3 is the alpha channel
+
+# Perform text extraction
+data = pytesseract.image_to_string(new_img, lang='eng')
+
+# Format
+data.strip()
+sentence = data[:-3]
+
+print(sentence)
+
+# format string
+captchaImgTitle = sentence.split('onto')[0].split('Drag the')[-1].strip()
+
+# print string
+print(captchaImgTitle)
+
+
+################################
+# Search for img in collection #
+################################
+
+imgName = captchaImgTitle.lower().replace(' ','_') + ".png"
+assert os.path.isfile(COLLECTION_FOLDER_PATH + imgName), "Image not found in collection"
+
+
+print(imgName)
+
+#########################
+# Detect img in Captcha #
+#########################
+
+method = cv2.TM_SQDIFF_NORMED
+
+# Read the images from the file
+small_image = cv2.imread(COLLECTION_FOLDER_PATH + imgName)
+large_image = cv2.imread(CAPTCHA_IMAGE_PATH)
+
+result = cv2.matchTemplate(small_image, large_image, method)
+
+# We want the minimum squared difference
+mn, _, mnLoc, _ = cv2.minMaxLoc(result)
+
+# Draw the rectangle:
+# Extract the coordinates of our best match
+MPx, MPy = mnLoc
+
+# Get the size of the template. This is the same size as the match.
+trows, tcols = small_image.shape[:2]
+
+# Get the coordinates of the template center on large_image
+centerPointx = MPx + int(tcols/2)
+centerPointy = MPy + int(trows/2)
+
+# Draw the rectangle on large_image
+cv2.rectangle(large_image, (MPx, MPy),
+              (MPx+tcols, MPy+trows), (0, 255, 255), 2)
+cv2.circle(large_image, (centerPointx, centerPointy),
+           radius=2, color=(0, 255, 255), thickness=-1)
+
+# Display the original image with the drawing
+cv2.imshow('output', large_image)
+
+
+#################
+# Return number #
+#################
+
+# Get the width of large_image
+largeWidth = large_image.shape[1]
+# Get the width of 1/4 large_image
+widthQuarter = largeWidth/4
+
+resultReturn = -1
+
+# Check the location of the centerPointx
+if centerPointx >= 0 and centerPointx < widthQuarter:
+    resultReturn = 0
+elif centerPointx >= widthQuarter and centerPointx < widthQuarter*2:
+    resultReturn = 1
+elif centerPointx >= widthQuarter*2 and centerPointx < widthQuarter*3:
+    resultReturn = 2
+elif centerPointx >= widthQuarter*3 and centerPointx < widthQuarter*4:
+    resultReturn = 3
+
+print("img n°", resultReturn+1, ": return", resultReturn)
+
+# The image is only displayed if we call this
+cv2.waitKey(0)
+cv2.destroyAllWindows()