Big_Data_Challenge_Books12_7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " Big_Data_Challenge_Books12.7.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "latex_envs": {
      "LaTeX_envs_menu_present": true,
      "autoclose": false,
      "autocomplete": true,
      "bibliofile": "biblio.bib",
      "cite_by": "apalike",
      "current_citInitial": 1,
      "eqLabelWithNumbers": true,
      "eqNumInitial": 1,
      "hotkeys": {
        "equation": "Ctrl-E",
        "itemize": "Ctrl-I"
      },
      "labels_anchors": false,
      "latex_user_defs": false,
      "report_style_numbering": false,
      "user_envs_cfg": false
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abednarz210/Big_Data_Challenge/blob/main/Big_Data_Challenge_Books12_7.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5WosHdRWdmpw",
        "outputId": "968c9d07-f878-4ffa-f4c0-61551ac5dfa4"
      },
      "source": [
        "import os\n",
        "# Find the latest version of spark 3.0  from http://www.apache.org/dist/spark/ and enter as the spark version\n",
        "# For example:\n",
        "# spark_version = 'spark-3.0.3'\n",
        "spark_version = 'spark-3.2.0'\n",
        "os.environ['SPARK_VERSION']=spark_version\n",
        "\n",
        "# Install Spark and Java\n",
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q http://www.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz\n",
        "!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "\n",
        "# Set Environment Variables\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = f\"/content/{spark_version}-bin-hadoop2.7\"\n",
        "\n",
        "# Start a SparkSession\n",
        "import findspark\n",
        "findspark.init()\n",
        "\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rIgn:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.152)] [Connecting to security.u\r                                                                               \rIgn:2 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.152)] [Connecting to security.u\r                                                                               \rHit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:5 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "Hit:6 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:8 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "Hit:9 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Hit:10 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:12 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:13 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:14 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Reading package lists... Done\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xISV-9eR6Yah",
        "outputId": "1882541b-6246-4bcd-c32a-72b394be831b"
      },
      "source": [
        "!wget https://jdbc.postgresql.org/download/postgresql-42.2.9.jar"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-12-08 04:37:58--  https://jdbc.postgresql.org/download/postgresql-42.2.9.jar\n",
            "Resolving jdbc.postgresql.org (jdbc.postgresql.org)... 72.32.157.228, 2001:4800:3e1:1::228\n",
            "Connecting to jdbc.postgresql.org (jdbc.postgresql.org)|72.32.157.228|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 914037 (893K) [application/java-archive]\n",
            "Saving to: ‘postgresql-42.2.9.jar’\n",
            "\n",
            "postgresql-42.2.9.j 100%[===================>] 892.61K  5.48MB/s    in 0.2s    \n",
            "\n",
            "2021-12-08 04:37:59 (5.48 MB/s) - ‘postgresql-42.2.9.jar’ saved [914037/914037]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEQFTWiV6g2h"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.appName(\"big_data_hw\").config(\"spark.driver.extraClassPath\",\"/content/postgresql-42.2.9.jar\").getOrCreate()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qik6uGSV9BWa",
        "outputId": "4f519481-03a8-4149-937c-b08392a28825"
      },
      "source": [
        "# Read in data from S3 Buckets\n",
        "from pyspark import SparkFiles\n",
        "url=\"https://db-6c14c855de480dadeb3a60a9f2e52024-s3-root-bucket.s3.us-west-2.amazonaws.com/amazon_reviews_us_Books_v1_02.tsv.gz\"\n",
        "spark.sparkContext.addFile(url)\n",
        "book_df = spark.read.csv(SparkFiles.get(\"amazon_reviews_us_Books_v1_02.tsv.gz\"), sep=\"\\t\", header=True, inferSchema=True)\n",
        "\n",
        "# Show DataFrame\n",
        "book_df.show(10)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|         US|   12076615| RQ58W7SMO911M|0385730586|     122662979|Sisterhood of the...|           Books|          4|            2|          3|   N|                N|this book was a g...|this boook was a ...| 2005-10-14|\n",
            "|         US|   12703090|  RF6IUKMGL8SF|0811828964|      56191234|The Bad Girl's Gu...|           Books|          3|            5|          5|   N|                N|           Fun Fluff|If you are lookin...| 2005-10-14|\n",
            "|         US|   12257412|R1DOSHH6AI622S|1844161560|     253182049|Eisenhorn (A Warh...|           Books|          4|            1|         22|   N|                N| this isn't a review|never read it-a y...| 2005-10-14|\n",
            "|         US|   50732546| RATOTLA3OF70O|0373836635|     348672532|Colby Conspiracy ...|           Books|          5|            2|          2|   N|                N|fine author on he...|Though she is hon...| 2005-10-14|\n",
            "|         US|   51964897|R1TNWRKIVHVYOV|0262181533|     598678717|The Psychology of...|           Books|          4|            0|          2|   N|                N|Execellent cursor...|Review based on a...| 2005-10-14|\n",
            "|         US|   31048862|R2F53LI9KK9MOY|0316769487|     862964341|The Catcher in th...|           Books|          4|            2|          2|   N|                N|   Interesting, fun.|My only complaint...| 2005-10-14|\n",
            "|         US|   53000124|R1KJ6MB7MRSQFF|0805076069|     145341889|Bait and Switch: ...|           Books|          4|            9|         11|   N|                N|I viewed this the...|This book is chil...| 2005-10-14|\n",
            "|         US|   29732693|R2XIM9LT335WHE|1581603681|     640542054|Opening Combinati...|           Books|          4|            3|         10|   N|                N|No Frills - Just ...|When looking for ...| 2005-10-14|\n",
            "|         US|   48541186|R1VE0FQQ0QTQJN|0300108834|     915891133|A Little History ...|           Books|          5|           16|         20|   N|                Y|Simple, entertain...|Never been much f...| 2005-10-14|\n",
            "|         US|   15280864|R1VKEE2NWSWDRU|0446531081|     880645124|Hour Game (King &...|           Books|          4|            0|          0|   N|                N|        A good read!|If you enjoy a we...| 2005-10-14|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SpbqhseC2JxQ",
        "outputId": "f2007338-633e-4462-9811-c260ceab85c0"
      },
      "source": [
        "#drop any null values\n",
        "dropna_df = book_df.dropna()\n",
        "dropna_df.show(10)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|         US|   12076615| RQ58W7SMO911M|0385730586|     122662979|Sisterhood of the...|           Books|          4|            2|          3|   N|                N|this book was a g...|this boook was a ...| 2005-10-14|\n",
            "|         US|   12703090|  RF6IUKMGL8SF|0811828964|      56191234|The Bad Girl's Gu...|           Books|          3|            5|          5|   N|                N|           Fun Fluff|If you are lookin...| 2005-10-14|\n",
            "|         US|   12257412|R1DOSHH6AI622S|1844161560|     253182049|Eisenhorn (A Warh...|           Books|          4|            1|         22|   N|                N| this isn't a review|never read it-a y...| 2005-10-14|\n",
            "|         US|   50732546| RATOTLA3OF70O|0373836635|     348672532|Colby Conspiracy ...|           Books|          5|            2|          2|   N|                N|fine author on he...|Though she is hon...| 2005-10-14|\n",
            "|         US|   51964897|R1TNWRKIVHVYOV|0262181533|     598678717|The Psychology of...|           Books|          4|            0|          2|   N|                N|Execellent cursor...|Review based on a...| 2005-10-14|\n",
            "|         US|   31048862|R2F53LI9KK9MOY|0316769487|     862964341|The Catcher in th...|           Books|          4|            2|          2|   N|                N|   Interesting, fun.|My only complaint...| 2005-10-14|\n",
            "|         US|   53000124|R1KJ6MB7MRSQFF|0805076069|     145341889|Bait and Switch: ...|           Books|          4|            9|         11|   N|                N|I viewed this the...|This book is chil...| 2005-10-14|\n",
            "|         US|   29732693|R2XIM9LT335WHE|1581603681|     640542054|Opening Combinati...|           Books|          4|            3|         10|   N|                N|No Frills - Just ...|When looking for ...| 2005-10-14|\n",
            "|         US|   48541186|R1VE0FQQ0QTQJN|0300108834|     915891133|A Little History ...|           Books|          5|           16|         20|   N|                Y|Simple, entertain...|Never been much f...| 2005-10-14|\n",
            "|         US|   15280864|R1VKEE2NWSWDRU|0446531081|     880645124|Hour Game (King &...|           Books|          4|            0|          0|   N|                N|        A good read!|If you enjoy a we...| 2005-10-14|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kx3hMnlAD5FA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d7a0e00e-9719-4a7e-9b29-c509486fc81f"
      },
      "source": [
        "#number of rows \n",
        "book_df.count()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3105520"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "book_df.head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rtTGcCiKXU6Y",
        "outputId": "fafad3da-d5aa-4c66-924f-63d4b8b4f9b0"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(marketplace='US', customer_id=12076615, review_id='RQ58W7SMO911M', product_id='0385730586', product_parent=122662979, product_title='Sisterhood of the Traveling Pants (Book 1)', product_category='Books', star_rating=4, helpful_votes=2, total_votes=3, vine='N', verified_purchase='N', review_headline='this book was a great learning novel!', review_body='this boook was a great one that you could learn from. it not only teaches the imponrtance of family and their values but it also deals with basic issues that teens and some kids even deal with.  this book is about 4 best friends who are for the first time in their lives spending their summer apart. one day they are all in one of the girls rooms and finds a pair of pants that were tucked away in her closet.  once all four  of them try them on they realize that there is really something special about these pants.  seeming as how all 4 girls are differnt shapes and sizes and somehow the pants fit all of them,  they realize that these pants were the start of something special.  immediatley following they decided to make up certian rules abut the pants such as you must write the best thing u did while wearing the pants over your summer on the right leg and also some silly things such as to \\\\\\\\\"never pick yuor nose while wearing the pants.\\\\\\\\\"  this book follows the girls throuh their summers in differnt places of the world and through all of the different obstacles that life takes them through. it can really teach you alot not only about what is going on around you but most imporntantly about yuorself.  i would give this book 4 stars and would reccommend it to anyone who seems the slihgtest bit interested.', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=12703090, review_id='RF6IUKMGL8SF', product_id='0811828964', product_parent=56191234, product_title=\"The Bad Girl's Guide to Getting What You Want\", product_category='Books', star_rating=3, helpful_votes=5, total_votes=5, vine='N', verified_purchase='N', review_headline='Fun Fluff', review_body=\"If you are looking for something to stimulate your brain, this isn't it.  However, if you are just looking for a good laugh, you'll enjoy The Bad Girl's Guide.  It's funny and light, and definitely a good way to pass a little bit of time.\", review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=12257412, review_id='R1DOSHH6AI622S', product_id='1844161560', product_parent=253182049, product_title='Eisenhorn (A Warhammer 40,000 Omnibus)', product_category='Books', star_rating=4, helpful_votes=1, total_votes=22, vine='N', verified_purchase='N', review_headline=\"this isn't a review\", review_body='never read it-a young relative idicated he liked it and somehow my name popped upon this--no more to say', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=50732546, review_id='RATOTLA3OF70O', product_id='0373836635', product_parent=348672532, product_title='Colby Conspiracy (Colby Agency)', product_category='Books', star_rating=5, helpful_votes=2, total_votes=2, vine='N', verified_purchase='N', review_headline='fine author on her A-game', review_body='Though she is honored to be Chicago Woman of the Year, Victoria Colby-Camp is more euphoric over the mental improvement that her son Jim has shown recently especially since he and Tasha North fell in love.  Jim was snatched almost twenty years ago when he was seven and turned into the killing Seth whose goal was to murder Victoria for abandoning him.  However, her elation would turn to despair if she knew Seth resurfaced and started to rape a pregnant Tasha.<br /><br />Former military strategist Daniel Marks is in town complements of the Colby Agency that wants to hire him.  Also in Chicago is Emily Hastings whose father a veteran homicide detective was murdered.  She finds letters linking her dad to Victoria, the woman\\'s long ago murdered first husband James, and her dad\\'s first partner Marelyn Rutland that confuses her.  Soon she will meet Daniel and they will be embroiled in the COLBY CONSPIRACY that goes back almost two decades ago.<br /><br />Though the subplots can become confusing at first, once the audience comprehends how this complex superb suspense thriller starts to come together, they will want more Colby Agency tales; (see FILES FROM THE COLBY AGENCY: THE BODYGUARD\\'S BABY PROTECTIVE CUSTODY).  The ensemble cast is solid as fans will feel with Victoria who has overcome so much tragedy, hope Jim \\\\\\\\\"defeats\\\\\\\\\" Seth with Tasha at his side, and root for Daniel and Emily to make it while wondering what really happened two decades ago.  A final twist marks a strong Webb of deceit tale that showcases a fine author on her A-game.<br /><br />Harriet Klausner', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=51964897, review_id='R1TNWRKIVHVYOV', product_id='0262181533', product_parent=598678717, product_title='The Psychology of Proof: Deductive Reasoning in Human Thinking', product_category='Books', star_rating=4, helpful_votes=0, total_votes=2, vine='N', verified_purchase='N', review_headline='Execellent cursor examination', review_body='Review based on a cursory examination by University of Phoenix students in Philosophy 251.  <br />We have found the book to be well organized, and detailed. The structure placed the information in an easy to read and presentable manner. <br />The use of strong and sound proofs by experiment and examples supported the overall logic of reasoning in a clear and concise manner. <br />We have found the book to be well thought out and laid out so the reader can learn and follow what is being explained. <br />', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=31048862, review_id='R2F53LI9KK9MOY', product_id='0316769487', product_parent=862964341, product_title='The Catcher in the Rye', product_category='Books', star_rating=4, helpful_votes=2, total_votes=2, vine='N', verified_purchase='N', review_headline='Interesting, fun.', review_body=\"My only complaint is that this book was a bit repetative, and Holden's vocabulary only consists of 4 words: sonuvabitch, goddamn, lousy and phony. It was still quite humorous and the actual story was interesting.<br /><br />It was easy to read and to relate to, so you should pick it up if you're looking for a fun read. Some people complain about the lack of a plot, but to understand this book you need to read deeper into it. It's actually a lot more than some annoying kid's list of complaints about life.<br /><br />The story is about Holden Caulfield's coming of age and realizing that growing up is not a process that can be stopped. He had a lousy childhood, as he puts it, and wants to save other children from going though the things he experienced in his life. He is telling the story from a mental institution, which he was placed in after returning home from the fourth Prep school he had been kicked out of. He flashes back to the events leading up to his arrival at his home and tells the story of those 3 days.\", review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=53000124, review_id='R1KJ6MB7MRSQFF', product_id='0805076069', product_parent=145341889, product_title='Bait and Switch: The (Futile) Pursuit of the American Dream', product_category='Books', star_rating=4, helpful_votes=9, total_votes=11, vine='N', verified_purchase='N', review_headline=\"I viewed this the opposite of Publisher's Weekly ...\", review_body='This book is chilling and depressing indeed, though very well written and sprinkled with frequent humor.  I was sort of surprised however by the comment in Publisher\\'s Weekly which said that \\\\\\\\\"PW suspects that many of those buyers will be disappointed. Ehrenreich can\\'t deliver the promised story because she never managed to get employed in the \\'midlevel corporate world\\' she wanted to analyze.\\\\\\\\\"  I never viewed that as being the \\\\\\\\\"promised story\\\\\\\\\" in the first place.  To me, the primary point of the book was to describe what it was like to actually try to ENTER that world in the first place rather than describing what life was like when she got there--and that she did.  In fact, I actually thought that she \\\\\\\\\"lucked out\\\\\\\\\" in a sense by having the bizarre, surreal experiences she did--not only did it make for more entertaining and more thought-provoking reading, but in my mind, if she HAD obtained a job right away, it would have taken away the whole raison d\\'etre of the book.  It would have been a completely different book in which she merely described her (presumably) boring mid-level corporate job, which probably would have been the same from day to day.  Thus, at least for purposes of writing the book, I think she was \\\\\\\\\"lucky\\\\\\\\\" that things worked out the way they did.  As for the reader who criticized her for not taking the \\\\\\\\\"jobs\\\\\\\\\" offered by AFLAC and Mary Kay, I thought those comments were bizarre.  I agree with Ms. Ehrenreich\\'s point in that \\\\\\\\\"jobs\\\\\\\\\" such as these in which the employer offers no salary, no benefits, no office, no help and essentially takes no risk whatsoever can hardly be considered jobs in the traditional sense she was seeking.  In any event, it\\'s really a pretty scary and depressing book, but one which I recommend.', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=29732693, review_id='R2XIM9LT335WHE', product_id='1581603681', product_parent=640542054, product_title='Opening Combination Padlocks: No Tools, No Problem', product_category='Books', star_rating=4, helpful_votes=3, total_votes=10, vine='N', verified_purchase='N', review_headline='No Frills - Just The Facts', review_body='When looking for &lt;a href=\\\\\\\\\"[...]\\\\\\\\\">lock picking books&lt;/a>, this one is very informative. It will give one great ideas on opening padlocks, but the language is very basic, no frills here. This is not a romance novel, it\\'s how to open locks without keys, and that\\'s it. You will learn what you need to here. But you should always seek out more knowledge.', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=48541186, review_id='R1VE0FQQ0QTQJN', product_id='0300108834', product_parent=915891133, product_title='A Little History of the World (Little Histories)', product_category='Books', star_rating=5, helpful_votes=16, total_votes=20, vine='N', verified_purchase='Y', review_headline='Simple, entertaining!', review_body='Never been much for enjoying history, but the style, brevity, and choice of subject matter come together very nicely in this book.', review_date='2005-10-14'),\n",
              " Row(marketplace='US', customer_id=15280864, review_id='R1VKEE2NWSWDRU', product_id='0446531081', product_parent=880645124, product_title='Hour Game (King & Maxwell)', product_category='Books', star_rating=4, helpful_votes=0, total_votes=0, vine='N', verified_purchase='N', review_headline='A good read!', review_body='If you enjoy a well told yarn of intrigue told so skillfully you are not able to predict the ending, a really good who-done-it, this is for you.', review_date='2005-10-14')]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#credit https://sparkbyexamples.com/pyspark/pyspark-distinct-to-drop-duplicates/\n",
        "\n",
        "distinctDF = book_df.distinct()\n",
        "print(\"Distinct count: \"+str(distinctDF.count()))\n",
        "distinctDF.show(truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WPrAjhr2Xpch",
        "outputId": "04a2f8da-2606-4a66-831a-960195e3ba92"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Distinct count: 3105520\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------------------------------------------------------------------------------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+\n",
            "|marketplace|customer_id|review_id     |product_id|product_parent|product_title                                                                                           |product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|review_headline                                                           |review_body                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------------------------------------------------------------------------------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+\n",
            "|US         |53066271   |R1001W6J9AVY60|0060194839|496072800     |Volumetrics: Feel Full on Fewer Calories                                                                |Books           |4          |7            |13         |N   |N                |Good Book...                                                              |I bought this book after I had my second child.  It gives you a list of filling, healthful foods to eat while watching your weight.  Very helpful.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |2001-05-23 |\n",
            "|US         |49255792   |R100JJ4ITUS5NK|0375760628|447916207     |Slave to Fashion: A Novel                                                                               |Books           |4          |1            |1          |N   |N                |Couldn't put it down                                                      |This is a really good satire, both of the fashion world and of the whole `Brit chick lit' genre. It's a simple story of the rise and fall of Katie Castle. I loved the interaction between Katie and Peggy Moss in the first section of the book. I admit that I found the first chapter or two slow, but by the time Katie and Peggy were in Paris, I was hooked. I thought Katie was actually a well rounded character. She wasn't a nice person, by any means, but I liked that about her - she appealed to my dark side. Katie could say and do things that I'd never dream of, so the book provided a good escape in that sense. I also liked that she did have something of a conscience (albeit not a well developed one), she did learn from her mistakes, and she had a degree of self awareness. I found myself wanting to see her succeed. The first part of the book developed the relationships well enough that I genuinely felt bad for her mistake, and I wanted to see her win back the boyfriend. Campbell manages to take a woman who is very different from most of the people who read the book, and yet make her someone we could relate to. Who hasn't messed up royally in their lives? She is able to find the insights and morals in the story. There are no profoundly original truths written here - it's all stuff we already know, on some level, but the reminders work. It's got just enough depth to be interesting, but it's not so serious that it comes across as pretentious.<br />The book wasn't absolutely perfect, of course. I thought some things happened too quickly and too easily, like the exposure of what Katie did, and even her ability to get back what she lost. I didn't like the ending - it was all just too easy. Some of the characters - particularly the men - were poorly developed. But those things are just minor complaints that keep the book from gaining full-fledged classic status. It's still better, funnier, and smarter than a lot of books available today. Worth a read!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |2004-04-13 |\n",
            "|US         |49394632   |R100M49CK52349|0553380966|199085075     |The Diamond Age: Or, a Young Lady's Illustrated Primer (Bantam Spectra Book)                            |Books           |5          |2            |4          |N   |N                |Diamond are Forever                                                       |I am not going to be long winded or reveal the high points and plot, I just want to say Neal  Stephenson did it again, not a sequel to Snow Crash but the same insane vain of wonderful, future world Sci-Fi. The basic tone of the book speaks volume of today socio-political environment and a very possible outcome. The plot and charters wind and copulate like few authors can manage. I enjoyed the way some of the side thoughts and charters were left as a statement unto them selves and not overly exploited or beat to pulp. This book will leave you thinking in a better vein and remembering bits and pieces for a time to come.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2003-09-17 |\n",
            "|US         |34342725   |R100MJQ2PCB17R|1570719276|875642269     |Living La Dolce Vita: Bring the Passion, Laughter and Serenity of Italy into Your Daily Life            |Books           |5          |9            |14         |N   |N                |this is a wonderful book!                                                 |I love this book!  If only for the chapter about food, this book is worth much more than the $15 I paid for it.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |2003-06-23 |\n",
            "|US         |41458372   |R100MQANUTTHLB|1561840580|557383629     |Pacts With the Devil: A Chronicle of Sex, Blasphemy and Liberation                                      |Books           |5          |6            |10         |N   |N                |Lucifer the lightbringer.  Prometheus the gift of fire.  Dionysos.  Shiva.|Great read.  Made me realize gods changed through time. It is quite enlightening how gods of one religion are twisted into demons by another.  I feel more affinity to Shiva and Dionysos anyhow and not some inaccessible hater off in the distance condemning everyone to hell for everything.      As far as the pacts and magic goes, I say to each his own.  Some people need to pray and beg.  Some need to sacrifice.  Some need to meditate.  Some need to visionquest.  Some just need that old fashioned black magic.      Lucifer is an amalgamation of dieties just like Jesus.  It's more like sibling rivaly than divinity vs profanity.  Ah, maybe it's meant to be a secret.  Several grimoires are included and if you are curious about the source of some of the material Check out \\\\\"Secret Lore of Magic\\\\\" by Idries Shah.  I got my copy for about $35 on ebay.  Sometimes prayers get answered and sometimes that crazy ritual just works.  It's really a matter of personal taste again isn't it?    Here's some simple but powerful magick I've searched long and hard to find:  -Om Shridaya Nama. Ganesh Mantra for Great Wealth.  -Pour Honey into a River and ask Oshun(Orisha) for Love or Money. Leave three pennies at the river's edge. Santeria.  -Go to a desolate crossroads alone and wait for the Crossroads spirit to give you a talent you desire.  Hoodoo.  -Vizualize what you want and see it manifest in your chest where the heart is and stamp it with authority into existence. Tantric Buddhism.    May you find the path that is right for you.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |2003-12-23 |\n",
            "|US         |35496930   |R101ETN2MK4NXL|014038572X|249148416     |The Outsiders                                                                                           |Books           |4          |1            |1          |N   |N                |decent book,                                                              |S.E Hinton's The Outsiders deals with the societal differences between the upper-class \\\\\"soc's\\\\\" and lower-class \\\\\"greasers,\\\\\" and the experiences that lead a boy to realize the superficial nature of these differences. Ponyboy Curtis, a greaser, is a high school student in a city where gangs are formed based on the part of town the gang members call home. Simply put, the socs have more than the greasers do, more money, more privilege, and therefore more status in the community. When a violent encounter erupts with Ponyboy and his best friend Johnny versus several drunken socs it exposes them to the series of wild events, Ponyboy experiences first-hand the uselessness of fighting and dying for an illogical division in class. Although there is one violent scene, the story really deals with how the people he meets in his young path towards emerging manhood affect Ponyboy's relationships and perceptions. His experiences with his brothers, his greaser friends, and even a few socs are all affected by his relationships with each other, and he comes to believe that if differences exist, they are only in the unwillingness each group has to changes established, habitual behavior.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |2002-12-15 |\n",
            "|US         |52946853   |R101V2DNSNS5SB|0312890168|121066554     |Ring of Swords                                                                                          |Books           |3          |7            |13         |N   |N                |A zig zag through an alien culture                                        |I must conclude that editing is a dying art; Eleanor Arnason's &quot;Ring of Swords&quot; is yet another SF novel of recent issue that could have benefited from another rewrite, and some judicious cutting. The length of the story is not supported by its weight, and fails utterly in the last hundred pages. The primary story (humans making first contact with an alien race) is far less vivid than the subplots--a marine world with intelligent squid, an alien who translates &quot;MacBeth&quot; magnificently into another culture. The hero is suitably ambiguous and anguished, the heroine is suitably plucky and determined, but it still doesn't add up to a dynamic read. It's yet another SF story where people sit around talking to one another too much.  Even Jane Austen sent her heroine out for a walk now and then.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |2001-03-16 |\n",
            "|US         |21392373   |R1027HXHGXMPHP|0451459318|809574707     |The Glasswrights' Test                                                                                  |Books           |5          |0            |3          |N   |N                |The Glasswright's Test                                                    |I loved this series! I like books with heroines in them and anymore it hard to find them that are not made to look good for the guys. These books are about a young girl who finds herself put into questionable situations and does what she believes is the best course of action. Being so young and naive has her at an disadvantage of not knowing what the true scheme of things are and she does not understand why everyone she loves and trusts now hates her for trying to do what she thought was right. This story shows how she grows and matures into a couragous, and strong woman, with the help of her new friends. I definitly reccomend these books, it makes for a fascinating world of reading. Looking forward to the fourth.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |2004-01-30 |\n",
            "|US         |51122057   |R102D7O1NVTNV5|0840731604|815489239     |Prewrath Rapture of the Church                                                                          |Books           |5          |31           |33         |N   |N                |Classic &quot;Must Read&quot;                                             |This was the first official book on the prewrath rapture, first published in 1990. Consequently, many people make every detail of Rosenthal's arguments in this book synonymous with the prewrath view itself. This puts an unfortunate amount of pressure on this book. This was the first formal presentation of the prewrath rapture, and since that time, the tangential details of this view have been refined. However, the basics outline of the view are solidly presented here, unchanged over time, and are -- in my view -- unshakeable. This is a classic volume that is a &quot;must read&quot; for anyone interested in various positions on the rapture. <br />What I like most about this book is its grace. The arguments for the prewrath timing of the rapture are extremely well presented, clear, and solid. But it is Rosenthal's grace in dissention and plea for unity among Christian brethren -- regardless of end-times view -- that is so impressive. Rosenthal himself was forced out of a position of authority based on his change in view from pretrib to prewrath and his experience has been since mirrored by many godly men and women in positions of authority. When did the timing of the rapture become a test of orthodoxy like the trinity, the blood atonment, and the resurrection? If I may borrow Paul's wise comment from two-thousand years ago, &quot;My brethren, it ought not be so.&quot; <br />The prewrath rapture is the only presentation of the rapture timing that answers all of the scriptural problems associated with rapture debates while taking the scriptures in their normative meaning. Had someone recognized this timing and put it into print before the pretrib rapture was formalized in the late 1800s (and, subsequently the midtrib and posttrib views), I firmly believe neither pretrib, nor midtrib, nor posttrib would need have been developed. Prewrath is the plain, obvious teaching of scripture. After having become extremely uncomfortable with the pretrib timing presented in evangelical circles, it is the timing that I, myself, determined on my own simply by studying the scriptures. Imagine my surprise when I found out later that someone else had come to the same conclusions and put them into print, and that the timing of which I had become convinced already had a name -- prewrath. <br />Bravo to Marvin Rosenthal for taking the bold step to put this view into print for the first time, regardless of the personal risk. His book remains now, as it will always be, a classic in the rapture debate that has formed a stepping stone on which so many others have built. <br />Other presentations of the classic prewrath view (I say &quot;classic&quot; because there are a growing number of writers using the &quot;prewrath&quot; mantle but who are actually presenting a post-70th-Week view) include Robert Van Kampen's The Rapture Question Answered: Plain &amp; Simple, Dave Bussard's Who Will Be Left Behind and When?, and my own Before God's Wrath: The Bible's Answer to the Timing of the Rapture.<br />H. L. Nigro, author of Before God's Wrath: The Bible's Answer to the Timing of the Rapture|2004-01-25 |\n",
            "|US         |49546072   |R102RZPC872T2K|014019536X|990968702     |Drawing Down the Moon: Witches, Druids, Goddess-Worshippers, and Other Pagans in America Today (Compass)|Books           |4          |2            |4          |N   |N                |Pagan World View                                                          |To me, only the 3 chapters in part one have to be read in succession, since the information deals with the prejudices facing Neo-Pagans, as well as the &quot;Pagan World View,&quot; which was good, but a bit scattered at  times...<br />The rest of the book works an encyclopedia featuring sketches of  different aspects of the Nro-Pagan movement, which can be read  independently of each other.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |2000-05-25 |\n",
            "|US         |34625226   |R102S6QR7283MK|0375414576|786383328     |My Life                                                                                                 |Books           |5          |18           |37         |N   |N                |The Making of a Hero                                                      |I admire the President and his wife for opening up their story to the public. The Clintons are an amazing truly American Family.Their values and the policies they set in motion have brought hope to America. Clinton's book is real and from the heart. You can see why he is so passionate about our country. His  story is unique and no matter what people say about his personal life, I find Bill and Hillary to be courageous modern and just leaders.Before the clintons people hated politicians.During the Clinton presidency we allowed one of our greatest leaders to be hunted down and prosecuted, judged for sins between him his wife, and ultimately his god. But today we are suppose to blindly support the current adminstrations policies, never mind knowing about Bush's personal life or more importantly his motivations are. How one earth was Clinton suppose to be at all these hearings leading to and then the actual impeachment and guard our country. But he did an amazing job anyway, he certainly didn't take a 2 month vacation. The death of Clintons father made him grow up too fast however it's better to reach for the sky when your father is in heaven and become president than to be handed a presidency because your Daddy was a president.Thankyou President Clinton and Sentor Clinton because of you our generation has hope.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2004-06-24 |\n",
            "|US         |52707995   |R103IPQL4B1W4L|0812019415|172654305     |Forgotten Calculus: A Refresher Course with Applications to Economics and Business                      |Books           |5          |3            |3          |N   |N                |This book was a life-saver for me.                                        |I would never have gotten through my first semester of Calc w/o this book.  The initial examples and problems were basic and then increased in complexity, allowing me to smoothly increase my skill level.  Would (and have) recommended it to my fellow students.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |1998-03-16 |\n",
            "|US         |20559144   |R103KHHJ8XAAQD|0060574534|855472227     |Love Monkey: A Novel                                                                                    |Books           |1          |15           |20         |N   |N                |er, uh, the, no, ah, um......                                             |Kyle Smith's LOVE MONKEY is so below par that I can't find the words to express my response to it. The story is one we've all read before, and not particularly memorable; the writing is not even MFA workshop-worthy; it's more college-level. I suppose that frat boys might think this book was good, but can't imagine anybody else would. I couldn't get into it, and couldn't get past the bad prose.  What I could do, thankfully, was put the book down, and move on to something better........                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |2004-03-03 |\n",
            "|US         |51938735   |R103WJ70C3BLON|0316557684|268518515     |Living Low-Carb: The Complete Guide to Long-Term Low-Carb Dieting                                       |Books           |4          |40           |96         |N   |Y                |A Useful Addition                                                         |As the author of &quot;The Gourmet Prescription: High Flavor Recipes for Lower Carbohydrate Diets&quot;, I welcome this addition to my low carb library. It contains useful information presented in an accessible manner and Ms. McCullough's perspective as a lay person has its merits. <br />As a fan of cauliflower, I particularly enjoyed her celebration of it as a low carb ally and potato substitute. Her Mashed &quot;Potatoes&quot; recipe is subtly appealing, although the long cooking and pureeing raise the glycemic index by affecting fiber content. Cauliflower lovers should also try Cauliflower-Green Bean Salad with Arame and Cauliflower-Tomato Salad with Almond Butter and Chives from &quot;The Gourmet Prescription&quot; which are bursting with flavor and leave most of the fiber intact.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |2000-09-21 |\n",
            "|US         |39720193   |R1043HE1FTE0QL|1588518868|711484556     |Thinking of Germany at Night                                                                            |Books           |5          |0            |1          |N   |N                |A novelist's  appreciation                                                |The eminent British historian and medieval mystery writer Dr. Paul C. Doherty has read &quot;Thinking of Germany at Night&quot; and calls it a &quot;most fascinating book.&quot;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2004-02-10 |\n",
            "|US         |46461860   |R104BP7P2K5GI1|1412028744|291850418     |The Good, the Fast and the Healthy                                                                      |Books           |4          |1            |1          |N   |N                |eating for healthy living                                                 |Jean's book is a stroke of genius. She has designed it for beginner cooks and advanced cooks alike.  Beginner cooks can follow the tasty recipes and enjoy being creative; the recipes are tried and tested.  Advanced cooks can use Jean's recipes as a guide and inspiration.   <br /> <br />Great book Jean!! <br /> <br />We are awaiting the next one!!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |2005-07-06 |\n",
            "|US         |51210590   |R104E43OX18ETL|0786004169|240670580     |Ecstasy (Arabesque)                                                                                     |Books           |5          |1            |2          |N   |N                |Gwynne Forster's best so far.                                             |I totally disagree with the reviewer from Evanston ,Illinois.  The traveling was a big part of the allure and appeal of the story.  The exotic locations enhanced the spice of the romantic settings.  I loved &quot;Against All Odds&quot; and wondered how Ms. Forster was going to top that, but in &quot;Ecstasy&quot; she does just that.  This lady has a way with the interaction between a man and a woman that brings the characters out of print into real life!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |1998-07-31 |\n",
            "|US         |51709412   |R104P5FCFG0MX8|081321081X|55129423      |Pius XII and the Holocaust: Understanding the Controversy                                               |Books           |5          |31           |34         |N   |N                |Pius without the Hagiography                                              |Jose Sanchez's Pius XII and the Holocaust is an excellent, scholarly presentation and summaryof the arguments surrounding the papacy of Eugenio Pacelli.  Sanchez has meticulously reviewed and revisited much of the source material used by defenders and critics of Pius.  His narrative is dispassionate and calm, always allowing the sources to speak for themselves.  He cites the major historians from all camps and clearly sets out their arguments with some comments as to the helpfulness of their claims.  At no point does he do other than that which he set out to do - namely allow the arguments space to be presented as clearly as possible.  Sanchez leaves it to the reader to make up their own mind.  For those who seek a \\\\\"final word\\\\\" or are waiting for the discovery of a \\\\\"smoking gun\\\\\", Sanchez will disappoint.  I believe the greatest strength of this work is providing the historian along with the general reader with a very useful handbook with which to navigate the continual stream of material about Pope Pius.  On subjects such as papal rhetoric and interpreting Vatican-speak, Sanchez does a great service.  Discussing the relationship of Pacelli and the Nazi regime is another example of skillful historical analysis.  It is not the last word or treatise on the subject, but it is a timely call to order on a subject that has more often than not been marred by polemic and name-calling among those who should know better.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2002-04-07 |\n",
            "|US         |25939712   |R104Q0OIP5ZH07|0679734481|808391018     |Success                                                                                                 |Books           |5          |12           |14         |N   |N                |an occasional masterpiece                                                 |Amis excells at playing nasty tricks on his readers, and &quot;Success&quot; is in many ways an emotional con game. As with all works of satire, the ultimate purpose of the novel is didactic. When &quot;Success&quot; works well (ie, when a reader is enough of a &quot;sucker&quot; to buy into Amis' conceit) it is a meditation on the ways we can be misled by pity, an audience-participation demonstration of the fallability of human sympathy. As such, it's a remarkably thought-provoking read.<br />That said, the success of &quot;Success&quot; is largely based on reader manipulation. There are a number of reasons why Amis' technique might not work for a particular reader - for instance, if they are easily offended, or if they don't find Amis' brand of humor funny, or (and this is absolutely vital) if they don't share the sympathy-for-the-underdog and corresponding lack-of-sympathy-for-the-overdog mentality upon which Amis' experiment depends. Without an emotional investment from the reader, &quot;Success&quot; reads as a heartlessly empty comedy, rife with cliche, riddled with needless sexism, racism, and homophobia, and featuring characters unique only in their dislikability. Once transformed by the gullability of the reader, however, &quot;Success&quot; becomes a fascinating and enlightening study of contemporary human nature.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2003-10-01 |\n",
            "|US         |51758907   |R1059YCV0CEI9T|0486274950|868416173     |Ancient Science Through the Golden Age of Greece                                                        |Books           |5          |18           |18         |N   |N                |A polished gem                                                            |I bought this book six years ago and have kept it within reach ever since. Open it up at a random page and you will find a gem of an essay on Babylonian astronomy, mummification techniques, the first etnographer, or  urban planning in ancient Greece. Sarton has managed to compress more than  thirty centuries of scientific history in 600 pages without becoming  superficial. He starts with the Babylonians and ends with the ancient  Greeks. Nothing is omitted. You will read about astronomy, astrology,  biology, mathematics, literature, history, crafts, technical achievements,  etnography, trade and economics, mining, government organization, law,  medicine, animal husbandry, and - of course - philosophy. <br />George Sarton  writes balanced, with authority and clarity, but first and foremost with  great love for his subject. His view is balanced and spiced with an  enormous number of delightful details. Did you know that the first Greek  colonists brought an urban planner with them? Or that citizens of Athens  received a tax credit in 473BC because of a windfall in mining revenues?  The Egyptian scientist who got it wrong but asked the right question will  have Sarton's sympathy. The Greek philosopher who made his name by reycling  ideas stolen from the Babylonians is condemned. <br />I should stop writing  before I get carried away. Let me say one more thing: it is truly  astonishing that this book has not become an instant bestseller.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |2000-06-20 |\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------------------------------------------------------------------------------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#drop columns to match schema\n",
        "\n",
        "distinctDF.drop(\"marketplace\",\"review_id \",\"lastname\", \"product_parent\", \"verified_purchase\", \"review_headline\", \"review_body\") \\\n",
        "    .printSchema()\n",
        "\n",
        "cols = (\"marketplace\",\"review_id \",\"lastname\", \"product_parent\", \"verified_purchase\", \"review_headline\", \"review_body\")\n",
        "\n",
        "distinctDF.drop(*cols) \\\n",
        "   .printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OeVKZ8HvZM9d",
        "outputId": "cd379b5f-00bc-4c9d-ea63-cb814d20b178"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- customer_id: integer (nullable = true)\n",
            " |-- review_id: string (nullable = true)\n",
            " |-- product_id: string (nullable = true)\n",
            " |-- product_title: string (nullable = true)\n",
            " |-- product_category: string (nullable = true)\n",
            " |-- star_rating: integer (nullable = true)\n",
            " |-- helpful_votes: integer (nullable = true)\n",
            " |-- total_votes: integer (nullable = true)\n",
            " |-- vine: string (nullable = true)\n",
            " |-- review_date: string (nullable = true)\n",
            "\n",
            "root\n",
            " |-- customer_id: integer (nullable = true)\n",
            " |-- review_id: string (nullable = true)\n",
            " |-- product_id: string (nullable = true)\n",
            " |-- product_title: string (nullable = true)\n",
            " |-- product_category: string (nullable = true)\n",
            " |-- star_rating: integer (nullable = true)\n",
            " |-- helpful_votes: integer (nullable = true)\n",
            " |-- total_votes: integer (nullable = true)\n",
            " |-- vine: string (nullable = true)\n",
            " |-- review_date: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    }
  ]
}