6 - GSC URL Inspection API.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TheMihirNaik/google-search-console-api-using-python/blob/main/6%20-%20GSC%20URL%20Inspection%20API.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Google Search Console API Class Code\n",
        "\n"
      ],
      "metadata": {
        "id": "f5b6TOsBFnFK"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "efSIDtetFb9p"
      },
      "outputs": [],
      "source": [
        "# Import required packages\n",
        "from oauth2client.client import OAuth2WebServerFlow\n",
        "from googleapiclient.discovery import build\n",
        "import httplib2\n",
        "import pandas as pd\n",
        "\n",
        "class GoogleSearchConsole:\n",
        "    def __init__(self, client_id, client_secret, oauth_scope, redirect_uri):\n",
        "        self.webmasters_service = self.authenticate(client_id, client_secret, oauth_scope, redirect_uri)\n",
        "\n",
        "    def authenticate(self, client_id, client_secret, oauth_scope, redirect_uri):\n",
        "      try:\n",
        "          # Authenticate with Google Search Console API\n",
        "          flow = OAuth2WebServerFlow(client_id, client_secret, oauth_scope, redirect_uri)\n",
        "          authorize_url = flow.step1_get_authorize_url()\n",
        "          print(\"Go to the following link in your browser: \" + authorize_url)\n",
        "          auth_code = input(\"Enter your Authorization Code here:\")\n",
        "          credentials = flow.step2_exchange(auth_code)\n",
        "          http = httplib2.Http()\n",
        "          creds = credentials.authorize(http)\n",
        "          webmasters_service = build('searchconsole', 'v1', http=creds)\n",
        "\n",
        "          return webmasters_service\n",
        "\n",
        "      except Exception as e:\n",
        "          # Handle authentication errors\n",
        "          print(\"Error during authentication:\", str(e))\n",
        "          return None\n",
        "\n",
        "    def get_site_list(self):\n",
        "        # Fetch the site data\n",
        "        site_list = self.webmasters_service.sites().list().execute()\n",
        "\n",
        "        all_sites = []\n",
        "        for each in site_list['siteEntry']:\n",
        "            all_sites.append(each['siteUrl'])\n",
        "\n",
        "        return all_sites\n",
        "\n",
        "    def fetch_search_console_data(self, website_url, start_date, end_date, dimensions, dimensionFilterGroups):\n",
        "        # Initialize an empty list to store the rows from the response\n",
        "        all_responses = []\n",
        "\n",
        "        # Initialize the start row to 0\n",
        "        start_row = 0\n",
        "\n",
        "        # Loop until all rows have been retrieved\n",
        "        while True:\n",
        "            # Build the request body for the API call\n",
        "            request_body = {\n",
        "                \"startDate\": start_date,\n",
        "                \"endDate\": end_date,\n",
        "                \"dimensions\": dimensions,\n",
        "                \"dimensionFilterGroups\": dimensionFilterGroups,\n",
        "                \"rowLimit\": 25000,\n",
        "                \"dataState\": \"final\",\n",
        "                'startRow': start_row\n",
        "            }\n",
        "\n",
        "            # Call the API with the request body\n",
        "            response_data = self.webmasters_service.searchanalytics().query(siteUrl=website_url, body=request_body).execute()\n",
        "\n",
        "            # Append the rows from the response to the all_responses list\n",
        "            for row in response_data['rows']:\n",
        "                # Create a temporary list to hold the values for the row\n",
        "                temp = []\n",
        "                # Extract the values for the keys (dimensions)\n",
        "                for key in row['keys']:\n",
        "                    temp.append(key)\n",
        "                # Extract the values for clicks, impressions, CTR, and position\n",
        "                temp.append(row['clicks'])\n",
        "                temp.append(row['impressions'])\n",
        "                temp.append(row['ctr'])\n",
        "                temp.append(row['position'])\n",
        "                # Append the row to the all_responses list\n",
        "                all_responses.append(temp)\n",
        "\n",
        "            # Update the start row to reflect the number of rows retrieved\n",
        "            start_row += len(response_data['rows'])\n",
        "\n",
        "            # Print a progress message\n",
        "            print(\"fetched up to \" + str(start_row) + \" rows of data\")\n",
        "\n",
        "            # Check if the number of rows retrieved is less than the row limit\n",
        "            if len(response_data['rows']) < 25000:\n",
        "                break\n",
        "\n",
        "        # Create a DataFrame from the all_responses list, with columns corresponding to the requested dimensions and metrics\n",
        "        df = pd.DataFrame(all_responses, columns=dimensions + ['clicks', 'impressions', 'ctr', 'position'])\n",
        "\n",
        "        # Return the DataFrame\n",
        "        return df\n",
        "\n",
        "    def url_inspection(self, page_url, website_url, languageCode):\n",
        "\n",
        "      request_body = {\"inspectionUrl\":page_url,\"siteUrl\": website_url,\"languageCode\":languageCode}\n",
        "\n",
        "      # I learned to make this query from Tobias Willmann - Here is the LinkedIn Post he shared. https://www.linkedin.com/feed/update/urn:li:activity:6893990001873670144/\n",
        "      json_output = self.webmasters_service.urlInspection().index().inspect(body=request_body).execute()\n",
        "\n",
        "      return json_output\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Declare Credentials"
      ],
      "metadata": {
        "id": "8GR5jIPBGRo0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "CLIENT_ID = \"1085687348720-cpabg6ttpsqjji0j6pgbkb8sbcdcp4nt.apps.googleusercontent.com\"\n",
        "CLIENT_SECRET = \"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\"\n",
        "OAUTH_SCOPE = \"https://www.googleapis.com/auth/webmasters.readonly\"\n",
        "REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'"
      ],
      "metadata": {
        "id": "eP08yeIdGQYe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Authenticate & Authortized"
      ],
      "metadata": {
        "id": "Je9KUEfhGXHC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an instance of the GoogleSearchConsole class\n",
        "gsc = GoogleSearchConsole(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE, REDIRECT_URI)"
      ],
      "metadata": {
        "id": "PC32_yeLGY24",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d3e41165-f007-4b85-8f85-8e5d495acac8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:oauth2client._helpers:__init__() takes at most 4 positional arguments (5 given)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Go to the following link in your browser: https://accounts.google.com/o/oauth2/v2/auth?client_id=1085687348720-cpabg6ttpsqjji0j6pgbkb8sbcdcp4nt.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fwebmasters.readonly&access_type=offline&response_type=code\n",
            "Enter your Authorization Code here:4/1AcvDMrD3DDXq9ZL-f9AjrYWH07dp2n_DxFFVDeKMXi3d9Vndh22_uKy8HD8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# List of Properties in your GSC Account"
      ],
      "metadata": {
        "id": "DwSntUUOfhMu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a list of sites\n",
        "sites = gsc.get_site_list()\n",
        "sites"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_g-Ev-nff5k",
        "outputId": "aeadc548-c22e-43e9-b495-3ac3757d6658"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['https://moneymonkdigital.com/',\n",
              " 'https://www.monocubed.com/',\n",
              " 'https://www.tridentpbi.in/']"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# URL Inspection API : Single URL"
      ],
      "metadata": {
        "id": "uvrjKEsVGqK6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "website_url = 'https://www.tridentpbi.in/'\n",
        "page_url = 'https://www.tridentpbi.in/products/industries/automobiles/'\n",
        "languageCode = 'en-US'"
      ],
      "metadata": {
        "id": "FHClBnMDGsmb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Run this code to get the indexing status of the page_url\n",
        "inspect_result = gsc.url_inspection(page_url, website_url, languageCode)"
      ],
      "metadata": {
        "id": "1m_W-FyAGxF6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# print the response\n",
        "inspect_result"
      ],
      "metadata": {
        "id": "o68iWtMUGyKx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bf94d683-fac2-4aa8-8081-7475ee434f60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'inspectionResult': {'inspectionResultLink': 'https://search.google.com/search-console/inspect?resource_id=https://www.tridentpbi.in/&id=CKpaLhMuCQu-FK2TBBP7gg&utm_medium=link&utm_source=api',\n",
              "  'indexStatusResult': {'verdict': 'PASS',\n",
              "   'coverageState': 'Submitted and indexed',\n",
              "   'robotsTxtState': 'ALLOWED',\n",
              "   'indexingState': 'INDEXING_ALLOWED',\n",
              "   'lastCrawlTime': '2024-07-28T00:06:38Z',\n",
              "   'pageFetchState': 'SUCCESSFUL',\n",
              "   'googleCanonical': 'https://www.tridentpbi.in/products/industries/automobiles/',\n",
              "   'userCanonical': 'https://www.tridentpbi.in/products/industries/automobiles/',\n",
              "   'referringUrls': ['https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/3-ply/',\n",
              "    'https://www.tridentpbi.in/'],\n",
              "   'crawledAs': 'MOBILE'},\n",
              "  'mobileUsabilityResult': {'verdict': 'VERDICT_UNSPECIFIED'}}}"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Option 1 : Scrape URLs from the Sitemap"
      ],
      "metadata": {
        "id": "q9cMZxu7HBPL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "def fetch_sitemap_urls(sitemap_url):\n",
        "\n",
        "    # Set up the headers\n",
        "    headers = {\n",
        "      \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36\"\n",
        "      }\n",
        "\n",
        "    # Send a GET request to the sitemap URL\n",
        "    response = requests.get(sitemap_url, headers=headers)\n",
        "    #print(response)\n",
        "\n",
        "    # Parse the XML response\n",
        "    soup = BeautifulSoup(response.text, 'xml')\n",
        "    #print(soup)\n",
        "\n",
        "    # Extract URLs, excluding image URLs\n",
        "    urls = [loc.text for loc in soup.find_all('loc') if 'image:loc' not in str(loc)]\n",
        "    #print(urls)\n",
        "\n",
        "    # Convert list to DataFrame with a specified column name\n",
        "    sitemap_urls_df = pd.DataFrame(urls, columns=['Sitemap_URLs'])\n",
        "\n",
        "    return sitemap_urls_df"
      ],
      "metadata": {
        "id": "G86_anTCmrKB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# run the function to get the URLs from sitemap\n",
        "sitemap_url = 'https://www.tridentpbi.in/page-sitemap.xml'\n",
        "\n",
        "# running the fuction\n",
        "sitemap_urls_df = fetch_sitemap_urls(sitemap_url)\n",
        "\n",
        "#print the df\n",
        "sitemap_urls_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 687
        },
        "collapsed": true,
        "id": "FkNtLnXtoekU",
        "outputId": "ddbf914a-db41-46bc-afcc-8a197fed1552"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                         Sitemap_URLs\n",
              "0                          https://www.tridentpbi.in/\n",
              "1               https://www.tridentpbi.in/contact-us/\n",
              "2   https://www.tridentpbi.in/products/corrugated-...\n",
              "3   https://www.tridentpbi.in/hot-coffee-cup-sleev...\n",
              "4   https://www.tridentpbi.in/products/corrugated-...\n",
              "5   https://www.tridentpbi.in/products/industries/...\n",
              "6   https://www.tridentpbi.in/products/corrugated-...\n",
              "7   https://www.tridentpbi.in/products/corrugated-...\n",
              "8   https://www.tridentpbi.in/products/industries/...\n",
              "9   https://www.tridentpbi.in/products/industries/...\n",
              "10  https://www.tridentpbi.in/products/corrugated-...\n",
              "11  https://www.tridentpbi.in/products/corrugated-...\n",
              "12  https://www.tridentpbi.in/products/corrugated-...\n",
              "13  https://www.tridentpbi.in/products/corrugated-...\n",
              "14  https://www.tridentpbi.in/products/industries/...\n",
              "15  https://www.tridentpbi.in/products/industries/...\n",
              "16  https://www.tridentpbi.in/products/industries/...\n",
              "17  https://www.tridentpbi.in/products/industries/...\n",
              "18  https://www.tridentpbi.in/products/industries/...\n",
              "19  https://www.tridentpbi.in/products/industries/...\n",
              "20     https://www.tridentpbi.in/products/industries/\n",
              "21  https://www.tridentpbi.in/products/corrugated-...\n",
              "22                https://www.tridentpbi.in/products/\n",
              "23          https://www.tridentpbi.in/infrastructure/\n",
              "24                https://www.tridentpbi.in/about-us/\n",
              "25                    https://www.tridentpbi.in/blog/"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fe9a6a72-509e-4eb0-b3f2-d17a3dccd7b2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sitemap_URLs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://www.tridentpbi.in/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www.tridentpbi.in/contact-us/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://www.tridentpbi.in/hot-coffee-cup-sleev...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>https://www.tridentpbi.in/products/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>https://www.tridentpbi.in/infrastructure/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>https://www.tridentpbi.in/about-us/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>https://www.tridentpbi.in/blog/</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fe9a6a72-509e-4eb0-b3f2-d17a3dccd7b2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fe9a6a72-509e-4eb0-b3f2-d17a3dccd7b2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fe9a6a72-509e-4eb0-b3f2-d17a3dccd7b2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c76a70a5-0d36-447a-8075-81451fe63dbd\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c76a70a5-0d36-447a-8075-81451fe63dbd')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c76a70a5-0d36-447a-8075-81451fe63dbd button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_2a970052-ec2a-4849-8a2f-4d8b2435b42d\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('sitemap_urls_df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_2a970052-ec2a-4849-8a2f-4d8b2435b42d button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('sitemap_urls_df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "sitemap_urls_df",
              "summary": "{\n  \"name\": \"sitemap_urls_df\",\n  \"rows\": 26,\n  \"fields\": [\n    {\n      \"column\": \"Sitemap_URLs\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 26,\n        \"samples\": [\n          \"https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\",\n          \"https://www.tridentpbi.in/products/industries/pharma-healthcare/\",\n          \"https://www.tridentpbi.in/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# limiting URLs up to 5 for this tutorial - comment out this code to get all the URLs\n",
        "urls_list = sitemap_urls_df.head(10)\n",
        "urls_list"
      ],
      "metadata": {
        "id": "ifzlYh60m4hn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "outputId": "372d9c79-0992-4b85-9c54-5778f4669e41"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        Sitemap_URLs\n",
              "0                         https://www.tridentpbi.in/\n",
              "1              https://www.tridentpbi.in/contact-us/\n",
              "2  https://www.tridentpbi.in/products/corrugated-...\n",
              "3  https://www.tridentpbi.in/hot-coffee-cup-sleev...\n",
              "4  https://www.tridentpbi.in/products/corrugated-...\n",
              "5  https://www.tridentpbi.in/products/industries/...\n",
              "6  https://www.tridentpbi.in/products/corrugated-...\n",
              "7  https://www.tridentpbi.in/products/corrugated-...\n",
              "8  https://www.tridentpbi.in/products/industries/...\n",
              "9  https://www.tridentpbi.in/products/industries/..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a1a53adc-b1f9-4363-bf0a-706a29985432\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sitemap_URLs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://www.tridentpbi.in/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www.tridentpbi.in/contact-us/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://www.tridentpbi.in/hot-coffee-cup-sleev...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a1a53adc-b1f9-4363-bf0a-706a29985432')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a1a53adc-b1f9-4363-bf0a-706a29985432 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a1a53adc-b1f9-4363-bf0a-706a29985432');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6e0d44b5-0218-46d2-ab22-cd8ac3ea040f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6e0d44b5-0218-46d2-ab22-cd8ac3ea040f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6e0d44b5-0218-46d2-ab22-cd8ac3ea040f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_eea9912e-56e6-46a1-9c97-679e05782be1\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('urls_list')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_eea9912e-56e6-46a1-9c97-679e05782be1 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('urls_list');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "urls_list",
              "summary": "{\n  \"name\": \"urls_list\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"Sitemap_URLs\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\",\n          \"https://www.tridentpbi.in/contact-us/\",\n          \"https://www.tridentpbi.in/products/industries/fruits-vegetables/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Fetch Indexing Status for all the URLs."
      ],
      "metadata": {
        "id": "smN3RBO8HsLB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def inspect_urls(urls_list, website_url, language_code='en-US'):\n",
        "  # Create an empty list to store the inspection results\n",
        "  inspection_results = []\n",
        "\n",
        "  # Iterate over the URLs in the dataframe\n",
        "  for index, row in urls_list.iterrows():\n",
        "      page_url = row['Sitemap_URLs']\n",
        "      print(page_url)\n",
        "\n",
        "      # Call the url_inspection function\n",
        "      inspect_result = gsc.url_inspection(page_url, website_url, language_code)\n",
        "\n",
        "      # Extract the desired data from the inspection result\n",
        "      json_output = inspect_result['inspectionResult']['indexStatusResult']\n",
        "      print(json_output)\n",
        "      print('fetch successful.')\n",
        "\n",
        "      # Append the inspection result to the list\n",
        "      inspection_results.append(json_output)\n",
        "\n",
        "  # Create a new dataframe with the inspection results\n",
        "  inspection_df = pd.DataFrame(inspection_results)\n",
        "\n",
        "  # Merge the original dataframe with the inspection results dataframe using the index\n",
        "  merged_df = pd.merge(urls_list, inspection_df, left_index=True, right_index=True)\n",
        "\n",
        "  return merged_df\n"
      ],
      "metadata": {
        "id": "97SJmW8ZHp5C",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Example usage\n",
        "# Assuming urls_list is already defined and contains the URLs\n",
        "inspection_results = inspect_urls(urls_list, website_url)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7qkpUt3-p3c4",
        "outputId": "69bb65a8-456d-4fe0-aff7-d17cb4f67bdf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://www.tridentpbi.in/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-10T12:07:14Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/', 'userCanonical': 'https://www.tridentpbi.in/', 'sitemap': ['https://www.tridentpbi.in/sitemap_index.xml'], 'referringUrls': ['https://tridentpbi.home.blog/', 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/duplex-white/', 'http://www.tridentpbi.in/', 'http://tridentpbi.in/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/contact-us/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-07-21T01:02:10Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/contact-us/', 'userCanonical': 'https://www.tridentpbi.in/contact-us/', 'sitemap': ['https://www.tridentpbi.in/sitemap_index.xml'], 'referringUrls': ['https://www.tridentpbi.in/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/corrugated-boxes-cardboard-cartons-exporters-india/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-08T12:11:24Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/corrugated-boxes-cardboard-cartons-exporters-india/', 'userCanonical': 'https://www.tridentpbi.in/products/corrugated-boxes-cardboard-cartons-exporters-india/', 'referringUrls': ['https://www.tridentpbi.in/', 'https://www.tridentpbi.in/contact-us/', 'http://tridentpbi.in/products/corrugated-box-cardboard-cartons/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/hot-coffee-cup-sleeves-manufacturer-exporter-in-india/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-07-27T23:07:13Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/hot-coffee-cup-sleeves-manufacturer-exporter-in-india/', 'userCanonical': 'https://www.tridentpbi.in/hot-coffee-cup-sleeves-manufacturer-exporter-in-india/', 'referringUrls': ['https://www.tridentpbi.in/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/5-ply/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-04T00:19:52Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/5-ply/', 'userCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/5-ply/', 'referringUrls': ['https://www.tridentpbi.in/', 'https://www.tridentpbi.in/products/industries/fruits-vegetables/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/industries/fruits-vegetables/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-06T14:13:38Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/industries/fruits-vegetables/', 'userCanonical': 'https://www.tridentpbi.in/products/industries/fruits-vegetables/', 'referringUrls': ['https://www.tridentpbi.in/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-05T04:24:50Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/', 'userCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/', 'referringUrls': ['https://www.tridentpbi.in/', 'https://www.tridentpbi.in/contact-us/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/3-ply/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-08T00:34:35Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/3-ply/', 'userCanonical': 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/3-ply/', 'referringUrls': ['https://www.tridentpbi.in/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-08-03T04:39:42Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/', 'userCanonical': 'https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/', 'referringUrls': ['https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/die-cut/', 'https://www.tridentpbi.in/', 'https://www.tridentpbi.in/products/corrugated-box-cardboard-cartons/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n",
            "https://www.tridentpbi.in/products/industries/storage-file-corrugated-box-manufacturer-exporter/\n",
            "{'verdict': 'PASS', 'coverageState': 'Submitted and indexed', 'robotsTxtState': 'ALLOWED', 'indexingState': 'INDEXING_ALLOWED', 'lastCrawlTime': '2024-06-26T16:32:34Z', 'pageFetchState': 'SUCCESSFUL', 'googleCanonical': 'https://www.tridentpbi.in/products/industries/storage-file-corrugated-box-manufacturer-exporter/', 'userCanonical': 'https://www.tridentpbi.in/products/industries/storage-file-corrugated-box-manufacturer-exporter/', 'referringUrls': ['https://www.tridentpbi.in/products/industries/storage-file-corrugated-box-manufacturer-exporter/'], 'crawledAs': 'MOBILE'}\n",
            "fetch successful.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#print this dataframe\n",
        "inspection_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 557
        },
        "id": "3p9fDb6gIW42",
        "outputId": "2ab92548-85c1-45b2-e6ee-2e9d0f45d263"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        Sitemap_URLs verdict  \\\n",
              "0                         https://www.tridentpbi.in/    PASS   \n",
              "1              https://www.tridentpbi.in/contact-us/    PASS   \n",
              "2  https://www.tridentpbi.in/products/corrugated-...    PASS   \n",
              "3  https://www.tridentpbi.in/hot-coffee-cup-sleev...    PASS   \n",
              "4  https://www.tridentpbi.in/products/corrugated-...    PASS   \n",
              "5  https://www.tridentpbi.in/products/industries/...    PASS   \n",
              "6  https://www.tridentpbi.in/products/corrugated-...    PASS   \n",
              "7  https://www.tridentpbi.in/products/corrugated-...    PASS   \n",
              "8  https://www.tridentpbi.in/products/industries/...    PASS   \n",
              "9  https://www.tridentpbi.in/products/industries/...    PASS   \n",
              "\n",
              "           coverageState robotsTxtState     indexingState  \\\n",
              "0  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "1  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "2  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "3  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "4  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "5  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "6  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "7  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "8  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "9  Submitted and indexed        ALLOWED  INDEXING_ALLOWED   \n",
              "\n",
              "          lastCrawlTime pageFetchState  \\\n",
              "0  2024-08-10T12:07:14Z     SUCCESSFUL   \n",
              "1  2024-07-21T01:02:10Z     SUCCESSFUL   \n",
              "2  2024-08-08T12:11:24Z     SUCCESSFUL   \n",
              "3  2024-07-27T23:07:13Z     SUCCESSFUL   \n",
              "4  2024-08-04T00:19:52Z     SUCCESSFUL   \n",
              "5  2024-08-06T14:13:38Z     SUCCESSFUL   \n",
              "6  2024-08-05T04:24:50Z     SUCCESSFUL   \n",
              "7  2024-08-08T00:34:35Z     SUCCESSFUL   \n",
              "8  2024-08-03T04:39:42Z     SUCCESSFUL   \n",
              "9  2024-06-26T16:32:34Z     SUCCESSFUL   \n",
              "\n",
              "                                     googleCanonical  \\\n",
              "0                         https://www.tridentpbi.in/   \n",
              "1              https://www.tridentpbi.in/contact-us/   \n",
              "2  https://www.tridentpbi.in/products/corrugated-...   \n",
              "3  https://www.tridentpbi.in/hot-coffee-cup-sleev...   \n",
              "4  https://www.tridentpbi.in/products/corrugated-...   \n",
              "5  https://www.tridentpbi.in/products/industries/...   \n",
              "6  https://www.tridentpbi.in/products/corrugated-...   \n",
              "7  https://www.tridentpbi.in/products/corrugated-...   \n",
              "8  https://www.tridentpbi.in/products/industries/...   \n",
              "9  https://www.tridentpbi.in/products/industries/...   \n",
              "\n",
              "                                       userCanonical  \\\n",
              "0                         https://www.tridentpbi.in/   \n",
              "1              https://www.tridentpbi.in/contact-us/   \n",
              "2  https://www.tridentpbi.in/products/corrugated-...   \n",
              "3  https://www.tridentpbi.in/hot-coffee-cup-sleev...   \n",
              "4  https://www.tridentpbi.in/products/corrugated-...   \n",
              "5  https://www.tridentpbi.in/products/industries/...   \n",
              "6  https://www.tridentpbi.in/products/corrugated-...   \n",
              "7  https://www.tridentpbi.in/products/corrugated-...   \n",
              "8  https://www.tridentpbi.in/products/industries/...   \n",
              "9  https://www.tridentpbi.in/products/industries/...   \n",
              "\n",
              "                                         sitemap  \\\n",
              "0  [https://www.tridentpbi.in/sitemap_index.xml]   \n",
              "1  [https://www.tridentpbi.in/sitemap_index.xml]   \n",
              "2                                            NaN   \n",
              "3                                            NaN   \n",
              "4                                            NaN   \n",
              "5                                            NaN   \n",
              "6                                            NaN   \n",
              "7                                            NaN   \n",
              "8                                            NaN   \n",
              "9                                            NaN   \n",
              "\n",
              "                                       referringUrls crawledAs  \n",
              "0  [https://tridentpbi.home.blog/, https://www.tr...    MOBILE  \n",
              "1                       [https://www.tridentpbi.in/]    MOBILE  \n",
              "2  [https://www.tridentpbi.in/, https://www.tride...    MOBILE  \n",
              "3                       [https://www.tridentpbi.in/]    MOBILE  \n",
              "4  [https://www.tridentpbi.in/, https://www.tride...    MOBILE  \n",
              "5                       [https://www.tridentpbi.in/]    MOBILE  \n",
              "6  [https://www.tridentpbi.in/, https://www.tride...    MOBILE  \n",
              "7                       [https://www.tridentpbi.in/]    MOBILE  \n",
              "8  [https://www.tridentpbi.in/products/corrugated...    MOBILE  \n",
              "9  [https://www.tridentpbi.in/products/industries...    MOBILE  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-766f0de6-15cb-4b3d-881e-6b8bb1ccfadc\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sitemap_URLs</th>\n",
              "      <th>verdict</th>\n",
              "      <th>coverageState</th>\n",
              "      <th>robotsTxtState</th>\n",
              "      <th>indexingState</th>\n",
              "      <th>lastCrawlTime</th>\n",
              "      <th>pageFetchState</th>\n",
              "      <th>googleCanonical</th>\n",
              "      <th>userCanonical</th>\n",
              "      <th>sitemap</th>\n",
              "      <th>referringUrls</th>\n",
              "      <th>crawledAs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://www.tridentpbi.in/</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-10T12:07:14Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/</td>\n",
              "      <td>https://www.tridentpbi.in/</td>\n",
              "      <td>[https://www.tridentpbi.in/sitemap_index.xml]</td>\n",
              "      <td>[https://tridentpbi.home.blog/, https://www.tr...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www.tridentpbi.in/contact-us/</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-07-21T01:02:10Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/contact-us/</td>\n",
              "      <td>https://www.tridentpbi.in/contact-us/</td>\n",
              "      <td>[https://www.tridentpbi.in/sitemap_index.xml]</td>\n",
              "      <td>[https://www.tridentpbi.in/]</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-08T12:11:24Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/, https://www.tride...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://www.tridentpbi.in/hot-coffee-cup-sleev...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-07-27T23:07:13Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/hot-coffee-cup-sleev...</td>\n",
              "      <td>https://www.tridentpbi.in/hot-coffee-cup-sleev...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/]</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-04T00:19:52Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/, https://www.tride...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-06T14:13:38Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/]</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-05T04:24:50Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/, https://www.tride...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-08T00:34:35Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>https://www.tridentpbi.in/products/corrugated-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/]</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-08-03T04:39:42Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/products/corrugated...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>PASS</td>\n",
              "      <td>Submitted and indexed</td>\n",
              "      <td>ALLOWED</td>\n",
              "      <td>INDEXING_ALLOWED</td>\n",
              "      <td>2024-06-26T16:32:34Z</td>\n",
              "      <td>SUCCESSFUL</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>https://www.tridentpbi.in/products/industries/...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[https://www.tridentpbi.in/products/industries...</td>\n",
              "      <td>MOBILE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-766f0de6-15cb-4b3d-881e-6b8bb1ccfadc')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-766f0de6-15cb-4b3d-881e-6b8bb1ccfadc button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-766f0de6-15cb-4b3d-881e-6b8bb1ccfadc');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5fcdcf0b-6d9a-4226-a32e-83128cc78556\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5fcdcf0b-6d9a-4226-a32e-83128cc78556')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5fcdcf0b-6d9a-4226-a32e-83128cc78556 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_7f1651e0-0fa6-4947-a792-37e61d52539b\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('inspection_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_7f1651e0-0fa6-4947-a792-37e61d52539b button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('inspection_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "inspection_results",
              "summary": "{\n  \"name\": \"inspection_results\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"Sitemap_URLs\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\",\n          \"https://www.tridentpbi.in/contact-us/\",\n          \"https://www.tridentpbi.in/products/industries/fruits-vegetables/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"verdict\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"PASS\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"coverageState\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"Submitted and indexed\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"robotsTxtState\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"ALLOWED\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"indexingState\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"INDEXING_ALLOWED\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"lastCrawlTime\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"2024-08-03T04:39:42Z\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pageFetchState\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"SUCCESSFUL\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"googleCanonical\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"userCanonical\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"https://www.tridentpbi.in/products/industries/processed-food-box/corrugated-pizza-box-manufacturer-exporter-in-india/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sitemap\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"referringUrls\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"crawledAs\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"MOBILE\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Export to Excel"
      ],
      "metadata": {
        "id": "T0ESLZARIEGv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Save the merged dataframe to a file\n",
        "inspection_results.to_excel('inspection_results.xlsx', index=False)"
      ],
      "metadata": {
        "id": "yEDqJulQIDzQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-jbVDhAgg_oY"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}