get_phrase_correction.py

import ourfunctions as f
import gc
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import re
from string import punctuation

train_df = pd.read_csv("./train.csv")
test_df = pd.read_csv("./test.csv")

print(train_df.shape)
print(test_df.shape)

## adjusting the nan value
train_df.fillna("", inplace=True)
test_df.fillna("", inplace=True)
train_df.info()



import enchant

import enchant
from enchant.checker import SpellChecker
d = enchant.DictWithPWL("en_US")
chkr = SpellChecker(d)


def create_unique_questions(train_df, test_df):
    words = list(
        set(
            list(set(train_df['question1'])) +
            list(set(train_df['question2'])) + list(set(test_df['question1']))
            + list(set(test_df['question2']))))
    unique_questions = pd.DataFrame(
        words, columns=['questions']).reset_index(drop=True)
    return unique_questions

unique_questions = create_unique_questions(train_df, test_df)


alreary_corrected = [
    "iphones",
    "ireland",
    "mustn",
    "linux",
    "wouldn",
    "videoes",
    "tv",
    "google",
    "memorise",
    "faught",
    "cena",
    "lesner",
    "hollywood",
    "anmount",
    "hoywood",
    "cantonese",
    "otherthan",
    "mumbai",
    "wikipedia",
    "textfields",
    "ajax",
    "pls",
    "couldn",
    "calcutta",
    "doesnt",
    "fIght",
    "txt",
    "whther",
    "feelns",
    "sudd",
    "stl",
    "india",
    "Plz",
    "engg",
    "eng",
    "olympians",
    "offence",
    "bulgarians",
    "siemens",
    "wasn",
    "clinton",
    "portland",
    "recognise",
    "adams",
    "didnt",
    "taylor",
    "youtube",
    "goverment",
    "korean",
    "paypal",
    "isn",
    "facebook",
    "mhz",
    "samsung",
    "womans",
    "german",
    "america",
    "mosquitos",
    "melbourne",
    "dj",
    "behaviour",
    "hasn",
    "phd",
    "aren",
    "ethernet",
    "uk",
    "realise",
    "brisbane",
    "organisation",
    "aftr",
    "russian",
    "nonpolar",
    "pc",
    "othet",
    "nokia",
    "boolean",
    "analyse",
    "centres",
    "ramadan",
    "latin",
    "weren",
    "immedietly",
    "bollywood",
    "conentration",
    "benifit",
    "oppurtunities",
    "filipino",
    "netflix",
    "indians",
    "opensource",
    "atlanta",
    "microsoft",
    "colour",
    "cse",
    "jane",
    "exsts",
    "persob",
    "centre",
    "radeon",
    "postgraduation",
    "suez",
    "illuminati",
    "analytics",
    "italian",
    "excercises",
    "favour",
    "smartphones",
    "shouldn",
    "didnot",
    "friday",
    "monday",
    "americans",
    "hasn",
    "michael",
    "verizon",
    "hitler",
    "fermi",
    "whatsapp",
    "messagess",
    "africa",
    "weakneses",
    "nikon",
    "capricorn",
    "romania",
    "favourite",
    "startups",
    "spanish",
    "preparegravitation",
    "compulsary",
    "workin",
    "syria",
    "immigants",
    "benedict",
    "legssss",
    "france",
    "watsup",
    "arya",
    "handjob",
    "europe",
    "shoud",
    "paypal",
    "upto",
    "paris",
    "sql",
    "hitman",
    "lagrangian",
    "dvd",
    "donald",
    "enigneering",
    "mightn",
    "defence",
    "iranian",
    "increse",
    "india",
    "hairloss",
    "volumetry",
    "americans",
    "quora",
    "eligiblty",
    "english",
    "indian",
    "bangalore",
    "emoji",
    "ielts",
    "ahmedabad",
    "frac",
    "sociall",
    "philippines",
    "java",
    "intraday",
    "mightn",
    "delhi",
    "saturn",
    "youtube",
    "noida",
    "lynda",
    "demonetisation",
    "html",
    "dissprove",
    "nlp",
    "nlp",
    "rollerblade",
    "vlc",
    "rolex",
    "november",
    "indians",
    "nflammatories",
    "absorps",
    "kat",
    "ibm",
    "centra",
    "centra",
    "uk",
    "pdf",
    "ebook",
    "sydney",
    "samsung",
    "usa",
    "traveller",
    "jaipur",
    "pablo",
    "ebay",
    "Ebay",
    "EBAY",
    "whatsapp",
    "imessage",
    "slary",
    "isis",
    "blow",
    "eu",
    "favourite",
    "reactjs",
    "pakistan",
    "stanford",
    "harvard",
    "wharton",
    "saturn",
    "existance",
    "gb",
    "poeple",
    "forex",
    "katrina",
    "decison",
    "snapchat",
    "rollerblade",
    "mba",
    "anime",
    "disney",
    "schengen",
    "singapore",
    "ramayan",
    "gmail",
    "madheshi",
    "germany",
    "instagram",
    "connecticut",
    "php",
    "reaso",
    "japanese",
    "gf",
    "mumbai",
    "robert",
    "linkedin",
    "maharashtrian",
    "bollywood",
    "enginnering",
    "rattata",
    "magikarp",
    "islam",
    "atleast",
    "atleast",
    "schengen",
    "engeneering",
    "casanova",
    "feelngs",
    "photoshop",
    "canada",
    "holland",
    "hollywood",
    "chelsea",
    "modernizaton",
    "instagrammer",
    "thailand",
    "chinese",
    "corrrect",
    "hillary",
    "china",
    "switzerland",
    "imrovement",
    "kms",
    "undergraduation",
    "qoura",
    "actived",
    "calender",
    "bestfriend",
    "dna",
    "latop",
    "permantley",
    "connectionn",
    "sylabus",
    "insititute",
    "sequrity",
    "addmision",
    "begineer",
    "wtiter",
    "litrate",
    "programing",
    "programmning",
    "demonitization",
    "intially",
    "unseccessful",
    "quikly",
]


i = 0
for q2 in unique_questions.questions.values:
    i += 1
    chkr.set_text(q2)
    for err in chkr:
        if not sum([c.isupper() for c in err.word]):
            error = err.word
            sugs = chkr.suggest(error)
            cond = True
            if len(sugs) > 2:
                cond = (len(sugs[0].split()) == 1) and ('-' not in sugs[1]) and (len(sugs) !=0)
            if cond  and (error not in alreary_corrected):
                print(q2)
                print(err.word)
                print(sugs, '\n\n')