diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..db16921 --- /dev/null +++ b/README.MD @@ -0,0 +1,32 @@ +# Captcha Solver for Ikabot +Login Captcha Solver for [Ikabot](https://github.com/physics-sp/ikabot) +## Source code + - The folder `collection` contains all the draggable icons + - `SolveCaptcha.py` contains a generic function to solve a login captcha + - `visualizeCaptchaSolver.py` can help you to see the process of detecting icons + - `testFunctionSolveCaptcha.py` shows you how to use `solveCaptcha()` function + - `ChopImg.py` contains a function that chops a Captcha image into several pieces + - `IkabotSimulation.py` is an example of use based on Ikabot +## Getting Started +**You will need to Install Tesseract** + +**1**. Install tesseract using windows installer available at: [https://github.com/UB-Mannheim/tesseract/wiki](https://github.com/UB-Mannheim/tesseract/wiki) + +**2**. Note the tesseract path from the installation. Default installation path at the time of this edit was: `C:\Users\USER\AppData\Local\Programs\Tesseract-OCR\tesseract.exe`. It may change so please check the installation path. + +**3**. `pip install pytesseract` + +**4**. Set the tesseract path (`TESSERACT_PATH`) in the script +## Contribution +All images have been listed in the collection at the time of this edit.
+If you want to add icons in the collection, you have to add the `.png` file in the folder `\collection`. + +## Screenshots +**Visualizer :** + +![Screenshot Captcha Solver](https://i.ibb.co/ByT6fp6/Captcha-Solver.png) + +**Simulation :** + +![Screenshot Captcha Solver](https://i.ibb.co/nC5kNWg/2021-07-27-09-07-05-Window.png) + diff --git a/SolveCaptcha.py b/SolveCaptcha.py new file mode 100644 index 0000000..6c46823 --- /dev/null +++ b/SolveCaptcha.py @@ -0,0 +1,118 @@ +import pytesseract +import cv2 +from PIL import Image +import os +import logging + +if os.name == 'nt': + TESSERACT_PATH = "C:/Users/soludev5/AppData/Local/Programs/Tesseract-OCR/tesseract.exe" # <------ /!\ CHANGE THIS /!\ + pytesseract.pytesseract.tesseract_cmd = TESSERACT_PATH + +NUMBER_OF_IMAGE_IN_CAPTCHA = 4 + +def extractText(textImg): + """This function returns a string which represents the name of the image that the text in `textImg` is describing. + Parameters + ---------- + textImg : PIL.Image + a picture that contains text, which can be OCR-ed by tesseract. + + Returns + ------- + captchaImgTitle : str + a string representing the name of the image that `textImg` is describing. + """ + + new_img = Image.new("RGB", textImg.size, (0, 0, 0)) + new_img.paste(textImg, mask=textImg.split()[3]) # 3 is the alpha channel + + # Perform text extraction + data = pytesseract.image_to_string(new_img, lang='eng') + + # Format + data.strip() + sentence = data[:-3] + + logging.info(sentence) + + # format string + captchaImgTitle = sentence.split('onto')[0].split('Drag the')[-1].strip() + + return captchaImgTitle + + +def solveCaptcha(textImg, captchaImg, collectionPath): + """This function return an integer in the range [0,3]. This integer represents the ordinal position of the image described textually in `textImg`, found in `collectionPath` within `captchaImg`. + Parameters + ---------- + textImg : PIL.Image + a picture that contains text, which can be OCR-ed by tesseract. + captchaImg : numpy.ndarray + a picture that contains within itself 4 pictures. This function will search for the index of the image described in `textImg` within this image and return it. + + Returns + ------- + resultReturn : int + an integer representing the index of the image described in `textImg` within `captchaImg`. + """ + ############## + # Read Text # + ############## + + captchaImgTitle = extractText(textImg) + + # print string + logging.info(captchaImgTitle) + + ################################ + # Search for img in collection # + ################################ + + imgName = captchaImgTitle.lower().replace(' ', '_') + ".png" + assert os.path.isfile(collectionPath + imgName), "Image not found" + + logging.info("Image found in collection :D") + + ######################### + # Detect img in Captcha # + ######################### + + method = cv2.TM_SQDIFF_NORMED + + # Read the images from the file + small_image = cv2.imread(collectionPath + imgName) + large_image = captchaImg + + result = cv2.matchTemplate(small_image, large_image, method) + + # We want the minimum squared difference + mn, _, mnLoc, _ = cv2.minMaxLoc(result) + + # Extract the coordinates of our best match + MPx, MPy = mnLoc + + # Get the size of the template. This is the same size as the match. + trows, tcols = small_image.shape[:2] + + # Get the coordinates of the template center on large_image + centerPointx = MPx + int(tcols/2) + centerPointy = MPy + int(trows/2) + + ################# + # Return number # + ################# + + # Get the width of large_image + largeWidth = large_image.shape[1] + # Get the width of 1/N large_image + widthQuarter = largeWidth/NUMBER_OF_IMAGE_IN_CAPTCHA + + # Check the location of the centerPointx + for i in range(0, NUMBER_OF_IMAGE_IN_CAPTCHA): + if centerPointx >= widthQuarter*i and centerPointx < widthQuarter*(i+1): + resultReturn = i + break + + logging.info("img n°", resultReturn+1) + + return resultReturn diff --git a/collection/apple.png b/collection/apple.png new file mode 100644 index 0000000..8fc0fbb Binary files /dev/null and b/collection/apple.png differ diff --git a/collection/balloon.png b/collection/balloon.png new file mode 100644 index 0000000..eefe574 Binary files /dev/null and b/collection/balloon.png differ diff --git a/collection/banana.png b/collection/banana.png new file mode 100644 index 0000000..f416661 Binary files /dev/null and b/collection/banana.png differ diff --git a/collection/bell.png b/collection/bell.png new file mode 100644 index 0000000..4b97f38 Binary files /dev/null and b/collection/bell.png differ diff --git a/collection/bicycle.png b/collection/bicycle.png new file mode 100644 index 0000000..097a44d Binary files /dev/null and b/collection/bicycle.png differ diff --git a/collection/bike.png b/collection/bike.png new file mode 100644 index 0000000..097a44d Binary files /dev/null and b/collection/bike.png differ diff --git a/collection/book.png b/collection/book.png new file mode 100644 index 0000000..2b27a94 Binary files /dev/null and b/collection/book.png differ diff --git a/collection/bottle.png b/collection/bottle.png new file mode 100644 index 0000000..a25057c Binary files /dev/null and b/collection/bottle.png differ diff --git a/collection/brush.png b/collection/brush.png new file mode 100644 index 0000000..282cbd4 Binary files /dev/null and b/collection/brush.png differ diff --git a/collection/candle.png b/collection/candle.png new file mode 100644 index 0000000..8a78048 Binary files /dev/null and b/collection/candle.png differ diff --git a/collection/carrot.png b/collection/carrot.png new file mode 100644 index 0000000..9726a0f Binary files /dev/null and b/collection/carrot.png differ diff --git a/collection/castle.png b/collection/castle.png new file mode 100644 index 0000000..64ab253 Binary files /dev/null and b/collection/castle.png differ diff --git a/collection/champagne_bottle.png b/collection/champagne_bottle.png new file mode 100644 index 0000000..a25057c Binary files /dev/null and b/collection/champagne_bottle.png differ diff --git a/collection/cherries.png b/collection/cherries.png new file mode 100644 index 0000000..bcf7a2d Binary files /dev/null and b/collection/cherries.png differ diff --git a/collection/cherry.png b/collection/cherry.png new file mode 100644 index 0000000..bcf7a2d Binary files /dev/null and b/collection/cherry.png differ diff --git a/collection/cloud.png b/collection/cloud.png new file mode 100644 index 0000000..88d874c Binary files /dev/null and b/collection/cloud.png differ diff --git a/collection/colored_pencils.png b/collection/colored_pencils.png new file mode 100644 index 0000000..7c0182c Binary files /dev/null and b/collection/colored_pencils.png differ diff --git a/collection/conputer.png b/collection/conputer.png new file mode 100644 index 0000000..4ed1789 Binary files /dev/null and b/collection/conputer.png differ diff --git a/collection/controller.png b/collection/controller.png new file mode 100644 index 0000000..f4395d1 Binary files /dev/null and b/collection/controller.png differ diff --git a/collection/crown.png b/collection/crown.png new file mode 100644 index 0000000..0f9a9b8 Binary files /dev/null and b/collection/crown.png differ diff --git a/collection/cup.png b/collection/cup.png new file mode 100644 index 0000000..c78476c Binary files /dev/null and b/collection/cup.png differ diff --git a/collection/dice.png b/collection/dice.png new file mode 100644 index 0000000..d8031ec Binary files /dev/null and b/collection/dice.png differ diff --git a/collection/donut.png b/collection/donut.png new file mode 100644 index 0000000..059caf2 Binary files /dev/null and b/collection/donut.png differ diff --git a/collection/droplet.png b/collection/droplet.png new file mode 100644 index 0000000..0168955 Binary files /dev/null and b/collection/droplet.png differ diff --git a/collection/flag.png b/collection/flag.png new file mode 100644 index 0000000..c96a80b Binary files /dev/null and b/collection/flag.png differ diff --git a/collection/flower.png b/collection/flower.png new file mode 100644 index 0000000..3b61827 Binary files /dev/null and b/collection/flower.png differ diff --git a/collection/fork.png b/collection/fork.png new file mode 100644 index 0000000..cd532e5 Binary files /dev/null and b/collection/fork.png differ diff --git a/collection/fried_egg.png b/collection/fried_egg.png new file mode 100644 index 0000000..59be093 Binary files /dev/null and b/collection/fried_egg.png differ diff --git a/collection/fruit.png b/collection/fruit.png new file mode 100644 index 0000000..2a23af9 Binary files /dev/null and b/collection/fruit.png differ diff --git a/collection/gamepad.png b/collection/gamepad.png new file mode 100644 index 0000000..3bd40ff Binary files /dev/null and b/collection/gamepad.png differ diff --git a/collection/glasses.png b/collection/glasses.png new file mode 100644 index 0000000..6597813 Binary files /dev/null and b/collection/glasses.png differ diff --git a/collection/globe.png b/collection/globe.png new file mode 100644 index 0000000..e47417d Binary files /dev/null and b/collection/globe.png differ diff --git a/collection/globus.png b/collection/globus.png new file mode 100644 index 0000000..e47417d Binary files /dev/null and b/collection/globus.png differ diff --git a/collection/guitar.png b/collection/guitar.png new file mode 100644 index 0000000..d564516 Binary files /dev/null and b/collection/guitar.png differ diff --git a/collection/ham_burger.png b/collection/ham_burger.png new file mode 100644 index 0000000..6137abd Binary files /dev/null and b/collection/ham_burger.png differ diff --git a/collection/hamburger.png b/collection/hamburger.png new file mode 100644 index 0000000..6137abd Binary files /dev/null and b/collection/hamburger.png differ diff --git a/collection/hat.png b/collection/hat.png new file mode 100644 index 0000000..6e058d0 Binary files /dev/null and b/collection/hat.png differ diff --git a/collection/heart.png b/collection/heart.png new file mode 100644 index 0000000..15bf2eb Binary files /dev/null and b/collection/heart.png differ diff --git a/collection/ice_cream.png b/collection/ice_cream.png new file mode 100644 index 0000000..2f98150 Binary files /dev/null and b/collection/ice_cream.png differ diff --git a/collection/jupiter.png b/collection/jupiter.png new file mode 100644 index 0000000..59344ab Binary files /dev/null and b/collection/jupiter.png differ diff --git a/collection/keys.png b/collection/keys.png new file mode 100644 index 0000000..77721e2 Binary files /dev/null and b/collection/keys.png differ diff --git a/collection/laptop.png b/collection/laptop.png new file mode 100644 index 0000000..4ed1789 Binary files /dev/null and b/collection/laptop.png differ diff --git a/collection/light_bulb.png b/collection/light_bulb.png new file mode 100644 index 0000000..100bc8f Binary files /dev/null and b/collection/light_bulb.png differ diff --git a/collection/magician_hat.png b/collection/magician_hat.png new file mode 100644 index 0000000..6e058d0 Binary files /dev/null and b/collection/magician_hat.png differ diff --git a/collection/magnet.png b/collection/magnet.png new file mode 100644 index 0000000..028dd72 Binary files /dev/null and b/collection/magnet.png differ diff --git a/collection/moon.png b/collection/moon.png new file mode 100644 index 0000000..dc49987 Binary files /dev/null and b/collection/moon.png differ diff --git a/collection/mug.png b/collection/mug.png new file mode 100644 index 0000000..c78476c Binary files /dev/null and b/collection/mug.png differ diff --git a/collection/orange.png b/collection/orange.png new file mode 100644 index 0000000..2a23af9 Binary files /dev/null and b/collection/orange.png differ diff --git a/collection/paintbrush.png b/collection/paintbrush.png new file mode 100644 index 0000000..282cbd4 Binary files /dev/null and b/collection/paintbrush.png differ diff --git a/collection/pens.png b/collection/pens.png new file mode 100644 index 0000000..7c0182c Binary files /dev/null and b/collection/pens.png differ diff --git a/collection/pirate_flag.png b/collection/pirate_flag.png new file mode 100644 index 0000000..c96a80b Binary files /dev/null and b/collection/pirate_flag.png differ diff --git a/collection/pirate_ship.png b/collection/pirate_ship.png new file mode 100644 index 0000000..0507074 Binary files /dev/null and b/collection/pirate_ship.png differ diff --git a/collection/planet.png b/collection/planet.png new file mode 100644 index 0000000..59344ab Binary files /dev/null and b/collection/planet.png differ diff --git a/collection/rainbow.png b/collection/rainbow.png new file mode 100644 index 0000000..1d10794 Binary files /dev/null and b/collection/rainbow.png differ diff --git a/collection/raindrop.png b/collection/raindrop.png new file mode 100644 index 0000000..0168955 Binary files /dev/null and b/collection/raindrop.png differ diff --git a/collection/sailboat.png b/collection/sailboat.png new file mode 100644 index 0000000..0507074 Binary files /dev/null and b/collection/sailboat.png differ diff --git a/collection/scissors.png b/collection/scissors.png new file mode 100644 index 0000000..1f7a276 Binary files /dev/null and b/collection/scissors.png differ diff --git a/collection/ship.png b/collection/ship.png new file mode 100644 index 0000000..0507074 Binary files /dev/null and b/collection/ship.png differ diff --git a/collection/star.png b/collection/star.png new file mode 100644 index 0000000..2084429 Binary files /dev/null and b/collection/star.png differ diff --git a/collection/sun.png b/collection/sun.png new file mode 100644 index 0000000..834886e Binary files /dev/null and b/collection/sun.png differ diff --git a/collection/top_hat.png b/collection/top_hat.png new file mode 100644 index 0000000..6e058d0 Binary files /dev/null and b/collection/top_hat.png differ diff --git a/collection/tophat.png b/collection/tophat.png new file mode 100644 index 0000000..6e058d0 Binary files /dev/null and b/collection/tophat.png differ diff --git a/collection/tree.png b/collection/tree.png new file mode 100644 index 0000000..dd7dd48 Binary files /dev/null and b/collection/tree.png differ diff --git a/collection/water_droplet.png b/collection/water_droplet.png new file mode 100644 index 0000000..0168955 Binary files /dev/null and b/collection/water_droplet.png differ diff --git a/collection/wine_bottle.png b/collection/wine_bottle.png new file mode 100644 index 0000000..a25057c Binary files /dev/null and b/collection/wine_bottle.png differ diff --git a/tests/ChopImg.py b/tests/ChopImg.py new file mode 100644 index 0000000..87441f8 --- /dev/null +++ b/tests/ChopImg.py @@ -0,0 +1,15 @@ +def chopImg(captchaImg): + # Chop img into pieces + NUMBER_OF_CHOPS = 4 + arrayImg = [] + width, height = captchaImg.size + chopsize = int(width/NUMBER_OF_CHOPS) + + for x0 in range(0, width, chopsize): + for y0 in range(0, height, chopsize): + box = (x0, y0, + x0+chopsize if x0+chopsize < width else width - 1, + y0+chopsize if y0+chopsize < height else height - 1) + arrayImg.append(captchaImg.crop(box)) + + return arrayImg \ No newline at end of file diff --git a/tests/IkabotSimulation.py b/tests/IkabotSimulation.py new file mode 100644 index 0000000..7014c57 --- /dev/null +++ b/tests/IkabotSimulation.py @@ -0,0 +1,87 @@ +from ChopImg import chopImg +import cv2 +from PIL import Image +import sys +import os +sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) ) +from SolveCaptcha import * + +COLLECTION_FOLDER_PATH = "collection/" + + +def saveImg(textImg, captchaImg, response): + name = extractText(textImg) + # convert from openCV2 to PIL. Notice the COLOR_BGR2RGB which means that + # the color is converted from BGR to RGB + color_coverted = cv2.cvtColor(captchaImg, cv2.COLOR_BGR2RGB) + pil_image = Image.fromarray(color_coverted) + + array = chopImg(pil_image) + + path = os.path.join(COLLECTION_FOLDER_PATH, name.lower().replace(' ','_') + ".png") + + img = array[int(response)-1] + img.save(path) + + print("Image saved in collection") + + +def sendToBot(textImg, captchaImg): + print("Sent to Telegram") + print("Please enter the number of the correct image (1, 2, 3 or 4)") + textImg.show() + cv2.imshow("captcha", captchaImg) + cv2.waitKey(0) + cv2.destroyAllWindows() + + +def getUserResponse(): + input1 = input() + return input1 + + +def main(): + global solvedByTelegram + print("The interactive captcha has been presented") + + while True: + TEXT_IMAGE_EXAMPLE_PATH = "tests/examples/txt2.png" + CAPTCHA_IMAGE_EXAMPLE_PATH = "tests/examples/img2.png" + + textImg = Image.open(TEXT_IMAGE_EXAMPLE_PATH) + textImg.load() + + captchaImg = cv2.imread(CAPTCHA_IMAGE_EXAMPLE_PATH) + + print("Trying to solve the captcha...") + result = solveCaptcha(textImg, captchaImg, COLLECTION_FOLDER_PATH) + print(result) + + if result == -1: + print("Can't solve the captcha. Please solve it via Telegram") + while True: + sendToBot(textImg, captchaImg) + response = getUserResponse() + if response == '': + continue + solvedByTelegram = True + break + else: + solvedByTelegram = False + + captcha_sent = {} + # captcha_sent = self.s.post('https://image-drop-challenge.gameforge.com/challenge/{}/en-GB'.format(challenge_id), json=data).json() + captcha_sent['status'] = "solved" + if captcha_sent['status'] == 'solved': + captchaSolved = True + if captchaSolved: + print("Captcha solved") + if solvedByTelegram: + saveImg(textImg, captchaImg, response) + break + else: + continue + + +if __name__ == '__main__': + main() diff --git a/tests/examples/img1.png b/tests/examples/img1.png new file mode 100644 index 0000000..8aa18a2 Binary files /dev/null and b/tests/examples/img1.png differ diff --git a/tests/examples/img2.png b/tests/examples/img2.png new file mode 100644 index 0000000..1f83d9e Binary files /dev/null and b/tests/examples/img2.png differ diff --git a/tests/examples/txt1.png b/tests/examples/txt1.png new file mode 100644 index 0000000..7756805 Binary files /dev/null and b/tests/examples/txt1.png differ diff --git a/tests/examples/txt2.png b/tests/examples/txt2.png new file mode 100644 index 0000000..2297866 Binary files /dev/null and b/tests/examples/txt2.png differ diff --git a/tests/testFunctionSolveCaptcha.py b/tests/testFunctionSolveCaptcha.py new file mode 100644 index 0000000..c3e0c28 --- /dev/null +++ b/tests/testFunctionSolveCaptcha.py @@ -0,0 +1,19 @@ +import cv2 +from PIL import Image +import sys +import os +sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) ) +from SolveCaptcha import solveCaptcha + +TEXT_IMAGE_PATH = "tests/examples/txt1.png" +CAPTCHA_IMAGE_PATH = "tests/examples/img1.png" +COLLECTION_FOLDER_PATH = "collection/" + +textImg = Image.open(TEXT_IMAGE_PATH) +textImg.load() + +captchaImg = cv2.imread(CAPTCHA_IMAGE_PATH) + +result = solveCaptcha(textImg, captchaImg, COLLECTION_FOLDER_PATH) + +print(result) diff --git a/tests/visualizeCaptchaSolver.py b/tests/visualizeCaptchaSolver.py new file mode 100644 index 0000000..184620f --- /dev/null +++ b/tests/visualizeCaptchaSolver.py @@ -0,0 +1,111 @@ +import pytesseract +import cv2 +from PIL import Image +import os + +TESSERACT_PATH = "C:/Users/soludev5/AppData/Local/Programs/Tesseract-OCR/tesseract.exe" +COLLECTION_FOLDER_PATH = "collection/" + +TEXT_IMAGE_PATH = "tests/examples/txt2.png" +CAPTCHA_IMAGE_PATH = "tests/examples/img2.png" + +############## +# Read Text # +############## + +pytesseract.pytesseract.tesseract_cmd = TESSERACT_PATH + +png = Image.open(TEXT_IMAGE_PATH) +png.load() # required for png.split() + +new_img = Image.new("RGB", png.size, (0, 0, 0)) +new_img.paste(png, mask=png.split()[3]) # 3 is the alpha channel + +# Perform text extraction +data = pytesseract.image_to_string(new_img, lang='eng') + +# Format +data.strip() +sentence = data[:-3] + +print(sentence) + +# format string +captchaImgTitle = sentence.split('onto')[0].split('Drag the')[-1].strip() + +# print string +print(captchaImgTitle) + + +################################ +# Search for img in collection # +################################ + +imgName = captchaImgTitle.lower().replace(' ','_') + ".png" +assert os.path.isfile(COLLECTION_FOLDER_PATH + imgName), "Image not found in collection" + + +print(imgName) + +######################### +# Detect img in Captcha # +######################### + +method = cv2.TM_SQDIFF_NORMED + +# Read the images from the file +small_image = cv2.imread(COLLECTION_FOLDER_PATH + imgName) +large_image = cv2.imread(CAPTCHA_IMAGE_PATH) + +result = cv2.matchTemplate(small_image, large_image, method) + +# We want the minimum squared difference +mn, _, mnLoc, _ = cv2.minMaxLoc(result) + +# Draw the rectangle: +# Extract the coordinates of our best match +MPx, MPy = mnLoc + +# Get the size of the template. This is the same size as the match. +trows, tcols = small_image.shape[:2] + +# Get the coordinates of the template center on large_image +centerPointx = MPx + int(tcols/2) +centerPointy = MPy + int(trows/2) + +# Draw the rectangle on large_image +cv2.rectangle(large_image, (MPx, MPy), + (MPx+tcols, MPy+trows), (0, 255, 255), 2) +cv2.circle(large_image, (centerPointx, centerPointy), + radius=2, color=(0, 255, 255), thickness=-1) + +# Display the original image with the drawing +cv2.imshow('output', large_image) + + +################# +# Return number # +################# + +# Get the width of large_image +largeWidth = large_image.shape[1] +# Get the width of 1/4 large_image +widthQuarter = largeWidth/4 + +resultReturn = -1 + +# Check the location of the centerPointx +if centerPointx >= 0 and centerPointx < widthQuarter: + resultReturn = 0 +elif centerPointx >= widthQuarter and centerPointx < widthQuarter*2: + resultReturn = 1 +elif centerPointx >= widthQuarter*2 and centerPointx < widthQuarter*3: + resultReturn = 2 +elif centerPointx >= widthQuarter*3 and centerPointx < widthQuarter*4: + resultReturn = 3 + +print("img n°", resultReturn+1, ": return", resultReturn) + +# The image is only displayed if we call this +cv2.waitKey(0) +cv2.destroyAllWindows()