lugnitdgp · goofynugtz · Oct 21, 2021 · Oct 27, 2021
diff --git a/RanjanRahul/app/__pycache__/query_processing.cpython-39.pyc b/RanjanRahul/app/__pycache__/query_processing.cpython-39.pyc
diff --git a/RanjanRahul/app/__pycache__/ranking.cpython-39.pyc b/RanjanRahul/app/__pycache__/ranking.cpython-39.pyc
diff --git a/RanjanRahul/app/app.py b/RanjanRahul/app/app.py
@@ -0,0 +1,59 @@
+from flask import Flask, render_template, request
+from flask_paginate import Pagination, get_page_args
+import pymongo
+
+app = Flask(__name__)
+
+@app.route('/')
+def entry():
+  return render_template('home.html')
+
+@app.route('/search_results/')
+def search_results():
+  connect_url = "mongodb://127.0.0.1:27017/"
+  client = pymongo.MongoClient(connect_url)
+
+  db = client.results
+
+  search_string = request.args.get('search')
+
+  # print("\n\n\nSearch: ", search_string)
+
+  query = []
+  query = db.search_results.find(
+    {'$text': {'$search': search_string, '$caseSensitive': False}}
+  )
+
+  print("\n\nQuery: ", query[0])
+
+  search_result = []
+
+  for doc in query:
+    exist = False
+    for result in search_result:
+      if result['title'] == doc['title'] or result['url'] == doc['url']:
+        exist = True
+        break
+
+    if exist == False:
+      search_result.append(doc)
+  print("\n\nSearch: ", search_result)
+
+  page, per_page, offset = get_page_args(page_parameter='page', per_page_parameter='per_page')
+
+  total = len(search_result)
+
+  pagination = Pagination(page=page, per_page=per_page, total=total, css_framework='bootstrap4')
+
+
+  return render_template('search.html', 
+    search_result = search_result[offset:offset + per_page],
+    page=page,
+    per_page=per_page,
+    pagination=pagination,
+    search_string=search_string
+  )
+
+
+if __name__ == '__main__':
+  app.run(debug=True)
diff --git a/RanjanRahul/app/static/fonts/sfmono-medium.ttf b/RanjanRahul/app/static/fonts/sfmono-medium.ttf
diff --git a/RanjanRahul/app/static/fonts/sfmono-regular.ttf b/RanjanRahul/app/static/fonts/sfmono-regular.ttf
diff --git a/RanjanRahul/app/static/main.css b/RanjanRahul/app/static/main.css
@@ -0,0 +1,57 @@
+@font-face {
+  font-family: "SF Mono Regular";
+  src: url("../static/fonts/sfmono-regular.ttf");
+}
+@font-face {
+  font-family: "SF Mono Medium";
+  src: url("../static/fonts/sfmono-medium.ttf");
+}
+
+html {
+	box-sizing: border-box;
+	font-size: 62.5%;
+	scroll-behavior: smooth;
+	scrollbar-width: none;
+}
+
+body {
+  margin: 0;
+  padding: 0;
+  font-size: 4rem;
+  margin: 0 5rem;
+  font-family: "SF Mono Regular";
+  /* background-color: beige; */
+  /* max-width: 102.4rem; */
+}
+
+body::-webkit-scrollbar{
+	display: none;
+}
+
+
+.flex-center {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.vp-size {
+  width: 100vw;
+  height: 100vh;
+}
+
+.search {
+  color: grey;
+  /* background-color: aquamarine; */
+  /* width: 800px; */
+}
+
+input {
+  appearance: none;
+  outline: 0;
+  border: 0;
+  font-family: 'SF Mono Medium';
+  font-size: 4rem;
+  /* max-width: max-content; */
+
+}
diff --git a/RanjanRahul/app/templates/base.html b/RanjanRahul/app/templates/base.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <link href="../static/main.css" rel="stylesheet" />
+  <title>search</title>
+</head>
+<body>
+  {% block content %}
+  {% endblock %}
+</body>
+</html>
diff --git a/RanjanRahul/app/templates/home.html b/RanjanRahul/app/templates/home.html
@@ -0,0 +1,18 @@
+{% extends 'base.html' %}
+{% block content %}
+
+<div class="flex-center vp-size">
+  <div class="search">
+    <span class="text">
+      search: 
+    </span>
+    <span class="cursor">
+      <form name="search" action="/search_results">
+        <input name="search" type="text" class="input-cursor" />
+        <button type="submit" class="btn btn-success">Search</button>
+      </form>
+    </span>
+  </div>
+</div>
+
+{% endblock %}
diff --git a/RanjanRahul/app/templates/search.html b/RanjanRahul/app/templates/search.html
@@ -0,0 +1,40 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="container">
+  <form class="" name="search" action="/search_results">
+    <div class="input-group">
+      <input type="text" class="form-control" name="search" placeholder="search...">
+      <button type="submit" class="btn btn-success">Search</button>
+    </div>
+  </form>
+
+  <div class="">
+    <p><b>Search result for '{{search_string}}'</b></p>
+  </div>
+
+  {% if search_result %}
+  <!-- {% print("Search page: ", search_result) %} -->
+  {% for link in search_result %}
+  <div class="">
+
+    <div class="">
+      <strong><a href="{{ link.url }}" target="_ blank">{{ link.title }}</a></strong>
+    </div>
+
+    <div class="col">
+      <a href="{{ link.url }}" target="_blank">{{ link.url }}</a>
+      <p>{{ link.description[:300] }}...</p>
+    </div>
+  </div>
+  {% endfor %}
+  {% else %}
+  <b>No results found</b>
+  {% endif %}
+
+  <div class="">
+    {{pagination.links}}
+  </div>
+</div>
+
+{% endblock %}
diff --git a/RanjanRahul/crawler.py b/RanjanRahul/crawler.py
@@ -0,0 +1,105 @@
+import pymongo
+import urllib.parse
+import requests
+from bs4 import BeautifulSoup
+import sys
+
+class Crawler():
+  client = pymongo.MongoClient("mongodb://127.0.0.1:27017/")
+  db = client.glugle
+
+  dissallowedLinks = []
+
+  def StartCrawl (self, url, depth):
+    complete_url = urllib.parse.urljoin(url, "/robots.txt")
+    print(complete_url)
+
+    try:
+      response = requests.get(complete_url)
+      print("Response: ", response)
+    except:
+      print("Failed to get response!")
+      self.Crawl(url, depth)
+
+    soup = BeautifulSoup(response.text, 'lxml')
+    robots_content = soup.find('p').text
+
+    links = robots_content.split()
+
+    for link in links:
+      if (link[0] == '/'):
+        self.dissallowedLinks.append(urllib.parse.urljoin(url, link))
+
+    print("Robots found and appended!")
+    self.Crawl(url, depth, self.dissallowedLinks)
+
+
+  def Crawl(self, url, depth, *dissallowedLinks):
+    try:
+      print(f"Crawling url: {url} at depth {depth}")
+      response = requests.get(url)
+    except:
+      print(f"Failed to perform GET req on {url}")
+      return
+
+    soup = BeautifulSoup(response.text, 'lxml')
+
+    try:
+      title = soup.find('title').text
+      description = ''
+
+      for tag in soup.findAll():
+        if tag.name == 'p':
+          description += tag.text.strip().replace('\n', '')
+
+    except:
+      print("Failed to retrieve title and desc.")
+      return
+
+    query = {
+      'url': url,
+      'title': title,
+      'description': description
+    }
+
+    search_results = self.db.search_results
+
+    search_results.insert_one(query)
+    search_results.create_index(
+      [
+        ('url', pymongo.TEXT),
+        ('title', pymongo.TEXT),
+        ('description', pymongo.TEXT)
+      ],
+      name = 'search_results',
+      default_language = 'english'
+    )
+
+    if depth == 0:
+      return
+
+    links =  soup.findAll('a')
+
+    for link in links:
+      try:
+        if link['href'] not in dissallowedLinks[0]:
+          if 'http' in link['href']:
+            self.Crawl(link['href'], depth-1, dissallowedLinks[0])
+
+          else:
+            link['href'] = urllib.parse.urljoin(url, link['href'])
+            self.Crawl(link['href'], depth-1, dissallowedLinks[0])
+
+      except:
+        print("No links retieved from page")
+        pass
+
+    self.client.close()
+
+
+crawler = Crawler()
+# crawler.StartCrawl("https://www.rottentomatoes.com", 2)
+
+crawler.StartCrawl(
+  sys.argv[1], int(sys.argv[2])
+)
diff --git a/RanjanRahul/requirements.txt b/RanjanRahul/requirements.txt
@@ -0,0 +1,10 @@
+beautifulsoup4==4.10.0
+bs4==0.0.1
+certifi==2021.10.8
+charset-normalizer==2.0.7
+idna==3.3
+lxml==4.6.3
+pymongo==3.12.1
+requests==2.26.0
+soupsieve==2.2.1
+urllib3==1.26.7