Notebook C. Patent Classification with CNN.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Notebook C: Patent Classification with CNN.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"3e8f651ae9c144849ded02196d727de1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_58d6c9b34a044184ab1a43becfdb418a","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_0f54639e9295433394eccba853256155","IPY_MODEL_fed7798063f243769a7d83448a73c57f"]}},"58d6c9b34a044184ab1a43becfdb418a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0f54639e9295433394eccba853256155":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_79bd5fef7bff45629805a626bc04ee8f","_dom_classes":[],"description":"Loop Through Embeddings: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9fa9f25ecfaa48349fa5387cecde8080"}},"fed7798063f243769a7d83448a73c57f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_5610bf1a97e6433aa87ad19b481d5093","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [18:02&lt;00:00, 216.52s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bc9dd2435d6b48eca811eae8ec25dd27"}},"79bd5fef7bff45629805a626bc04ee8f":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"9fa9f25ecfaa48349fa5387cecde8080":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5610bf1a97e6433aa87ad19b481d5093":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"bc9dd2435d6b48eca811eae8ec25dd27":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5bbe24e6da984306866714087af988c3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9a0f3bfef12845fd8cf29a28e49bc134","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a531a4f3e85d43249a8bde93793253f1","IPY_MODEL_4232e2b7129d440ba2c3e08e69c1b0e4"]}},"9a0f3bfef12845fd8cf29a28e49bc134":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a531a4f3e85d43249a8bde93793253f1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_f0f80855875b4ab198fa60d0931966a0","_dom_classes":[],"description":"Cross-Validating: 100%","_model_name":"FloatProgressModel","bar_style":"","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c4d9c9d56c9a4bb79a8360459bd0fe48"}},"4232e2b7129d440ba2c3e08e69c1b0e4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ae219fc95f6b4711ab73b73616e0e76c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [02:25&lt;00:00, 31.14s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_87241e451f3b42c8a8e69fa02416ee6f"}},"f0f80855875b4ab198fa60d0931966a0":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c4d9c9d56c9a4bb79a8360459bd0fe48":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ae219fc95f6b4711ab73b73616e0e76c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"87241e451f3b42c8a8e69fa02416ee6f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0c3d2b30e6d047a9975e8b89a44d84fc":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_7ef5642ab14b45edae69887b45830869","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_5184d424658948ec8996d385f06d349d","IPY_MODEL_890580f3713c409796361203cdd0d579"]}},"7ef5642ab14b45edae69887b45830869":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5184d424658948ec8996d385f06d349d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_a75bc7f1fb254420b4681e4f45ec907e","_dom_classes":[],"description":"Cross-Validating: 100%","_model_name":"FloatProgressModel","bar_style":"","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_66bf4fc292e0443b989c3cf7ad9327b4"}},"890580f3713c409796361203cdd0d579":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_b94664e9d02145579be95b2bcd30dee5","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [01:31&lt;00:00, 18.52s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_cebb89edeae540e79272a4816e228439"}},"a75bc7f1fb254420b4681e4f45ec907e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"66bf4fc292e0443b989c3cf7ad9327b4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b94664e9d02145579be95b2bcd30dee5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"cebb89edeae540e79272a4816e228439":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ab296e24299840e38e63b2b456486309":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_d9168727aaad4dc0991bc9ea489174e6","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_97c3e6663e4d4de0b07f208a6aa33be2","IPY_MODEL_28851d0321114672a69c947555861ddb"]}},"d9168727aaad4dc0991bc9ea489174e6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"97c3e6663e4d4de0b07f208a6aa33be2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_35043a05b7304abf95151b7f7f5d63f3","_dom_classes":[],"description":"Cross-Validating: 100%","_model_name":"FloatProgressModel","bar_style":"","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4d1782236f174497b1cd59d85f9c81bc"}},"28851d0321114672a69c947555861ddb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_10fee2087b634b53899ddf0510919f96","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [04:45&lt;00:00, 53.70s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8d6ded4fbec241eeab7e423a25f1ce58"}},"35043a05b7304abf95151b7f7f5d63f3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"4d1782236f174497b1cd59d85f9c81bc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"10fee2087b634b53899ddf0510919f96":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"8d6ded4fbec241eeab7e423a25f1ce58":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d3ec62e1e11b4e5fa47041f1d8e9240f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_b0a30546cbe64a85b1ad176937463ed8","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_7d3442e9b3c5465d809546d2f0f10e7d","IPY_MODEL_ae9d358158554ea1b434e2d3f7109594"]}},"b0a30546cbe64a85b1ad176937463ed8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7d3442e9b3c5465d809546d2f0f10e7d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_414bdc2a145b48b8b2b48da2125b0f76","_dom_classes":[],"description":"Cross-Validating: 100%","_model_name":"FloatProgressModel","bar_style":"","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a4751bc3c2884df782102bac010eb242"}},"ae9d358158554ea1b434e2d3f7109594":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c901457a48a94ab59f96f8c5dd8a5089","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [01:35&lt;00:00, 18.94s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_147748405abb44ec8c11ca0fbbf8d7d5"}},"414bdc2a145b48b8b2b48da2125b0f76":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"a4751bc3c2884df782102bac010eb242":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c901457a48a94ab59f96f8c5dd8a5089":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"147748405abb44ec8c11ca0fbbf8d7d5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"894fbe5b51d147e09af22d4a22d8ab60":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_189bf04254b54a2488ffd1cc3b1e3fb3","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_f8036fd40c9f474180d7246532dd78e3","IPY_MODEL_3767d93432cf477092339dd93c3693d5"]}},"189bf04254b54a2488ffd1cc3b1e3fb3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f8036fd40c9f474180d7246532dd78e3":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_530e8f42828443889129923cb1efeb17","_dom_classes":[],"description":"Cross-Validating: 100%","_model_name":"FloatProgressModel","bar_style":"","max":5,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":5,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c1553306c7334c1a88186f745ae9970f"}},"3767d93432cf477092339dd93c3693d5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_59da9f4cdf264b09a7c587eef5cc08bf","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 5/5 [03:57&lt;00:00, 47.40s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_15c4e7f41b67431fa91814cc400dabe3"}},"530e8f42828443889129923cb1efeb17":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c1553306c7334c1a88186f745ae9970f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"59da9f4cdf264b09a7c587eef5cc08bf":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"15c4e7f41b67431fa91814cc400dabe3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"FYqkXcokQGt1"},"source":["# **Notebook C**: Patent Classification with CNN \n","---- \n","\n"]},{"cell_type":"markdown","metadata":{"id":"gAf0Z4gHQOAC"},"source":["# C.1. Load Packages \n","---"]},{"cell_type":"code","metadata":{"id":"tPa6y5QHyKxn","executionInfo":{"status":"ok","timestamp":1626215557060,"user_tz":420,"elapsed":2712,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# General Packages #\n","import os\n","import pandas as pd\n","import numpy as np\n","\n","# Load TQDM to Show Progress Bars #\n","from tqdm import tqdm\n","from tqdm.notebook import tqdm as tqdm_notebook\n","\n","from sklearn.model_selection import train_test_split, StratifiedShuffleSplit, StratifiedKFold\n","from sklearn.metrics import accuracy_score, roc_auc_score, precision_score, recall_score, f1_score, confusion_matrix\n","\n","# Keras Packages #\n","\n","import tensorflow as tf\n","\n","from keras.preprocessing.text import Tokenizer\n","from keras.preprocessing.sequence import pad_sequences\n","\n","from keras.models import Sequential\n","from keras import layers\n","from keras.layers import Dense, Dropout, Activation, GlobalMaxPooling1D, LSTM, Bidirectional, BatchNormalization\n","from keras.layers.convolutional import Conv1D, MaxPooling1D\n","\n","\n","from textblob import TextBlob\n","\n","import zipfile"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5TylwAeOzefE","executionInfo":{"status":"ok","timestamp":1626215557272,"user_tz":420,"elapsed":214,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}},"outputId":"cb7c57f1-5a68-4d85-edc7-4c30a0d17091"},"source":["import nltk\n","nltk.download('punkt')"],"execution_count":2,"outputs":[{"output_type":"stream","text":["[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"ENZ0VK8NySIy","executionInfo":{"status":"ok","timestamp":1626215557273,"user_tz":420,"elapsed":7,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Turn off warnings, just to avoid pesky messages that might cause confusion here\n","# Remove when testing your own code #\n","import warnings\n","warnings.filterwarnings(\"ignore\")"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uOs_vBkMyTSV","executionInfo":{"status":"ok","timestamp":1626215557282,"user_tz":420,"elapsed":15,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}},"outputId":"77c229ae-c3de-4317-e04c-704c7c6c573d"},"source":["# Mount Personal Google Drive on own Machine -- You have to follow the link to log in #\n","from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"qvcRsIMhydat"},"source":["# C.2. Load Training Data ##\n","---------------- \n","\n","We are going to use the data on the Google drive. This is in a csv file, and so we are going to load the data as a dataframe, and then convert the main data (Patent Ids, Indicator for AI / Non-AI, Patent Abstract) from a Pandas DataFrame to a list (which is more easily used in later sections). "]},{"cell_type":"code","metadata":{"id":"db2hTpK8yVV7","executionInfo":{"status":"ok","timestamp":1626215557514,"user_tz":420,"elapsed":245,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Change to Working Directory with Training Data # \n","os.chdir(\"/content/drive/MyDrive/USPTO AI Patent Classification/\")\n","\n","# Load Training Data #\n","TrainingData = pd.read_csv(\"./Training Data/4K Patents - AI 20p.csv\")\n","\n","# Store Data in Lists for Text Classification #\n","IDs = np.array(TrainingData['app number'].values.tolist())\n","Abstract_Text = TrainingData['abstract'].values.tolist()\n","Classes = TrainingData['actual'].values.tolist()"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"NC6OcdTMyf4E","executionInfo":{"status":"ok","timestamp":1626215557515,"user_tz":420,"elapsed":3,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# We need to use the Keras based tokenizer. We also need to define a custom text cleaner function which is going \n","# to stem the words (this is what we have been doing for all of the models)\n","\n","n_words = 2000\n","\n","# Define Tokenizer Function #\n","\n","tokenizer = Tokenizer(num_words=n_words, lower = True,\n","                        filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^`{|}~\\t\\n',\n","                        char_level=False)\n","                        \n","\n","# Define String Cleaner Function \n","def string_cleaner(str_input):\n","    blob = TextBlob(str_input.lower())\n","    tokens = blob.words\n","    words = [token.stem() for token in tokens]\n","    # If you want to use non-stemmed (Full words not just base) you can use the comment below\n","    #words = [token for token in tokens]\n","    return words\n"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"Avx5mdheTI9G","executionInfo":{"status":"ok","timestamp":1626215569145,"user_tz":420,"elapsed":11632,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Convert the Abstracts to a Keras Format #\n","\n","Abstracts_Lemmatized = []\n","for x in Abstract_Text:\n","    segments = string_cleaner(x)\n","    Abstracts_Lemmatized.append( \" \".join(segments))\n","\n","Abstracts = Abstracts_Lemmatized\n","\n","# Fit the abstracts to obtain the word index which will be subsequently used # \n","tokenizer.fit_on_texts(Abstracts)\n","\n","vocab_size = len(tokenizer.word_index) + 1"],"execution_count":7,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"1ToHyYyAoyvm"},"source":["Once we have the list of words that occur in our corpus of abstracts (i.e. word index), then we can try to map those words to embedding vectors. Below we define the functions that will go through each of the words in our word index and extract the coresponding embedding vector and save it to an embedding matrix that will be used as a layer in a subsequent convolutional neural network (CNN) model. "]},{"cell_type":"code","metadata":{"id":"YzM8h0SPS8j1","executionInfo":{"status":"ok","timestamp":1626215569147,"user_tz":420,"elapsed":4,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Create Embedding Matrix by Loading Embedding File and Mapping it to Word Index #\n","\n","def create_embedding_matrix(filepath, word_index, embedding_dim):\n","    vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n","    embedding_matrix = np.zeros((vocab_size, embedding_dim))\n","\n","    with open(filepath, encoding='utf-8') as f:\n","        for line in f:\n","            word, *vector = line.split()\n","            if word in word_index:\n","                idx = word_index[word]\n","                try:\n","                  embedding_matrix[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]\n","                except:\n","                  embedding_matrix[idx] = np.array([0]*embedding_dim, dtype=np.float32)[:embedding_dim]\n","    return embedding_matrix\n","\n","# Function to Create Empty Embedding Matrix -- For model without embeddings #\n","def create_empty_matrix(word_index, embedding_dim):\n","    vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n","    embedding_matrix = np.ones((vocab_size, embedding_dim))\n","\n","    return embedding_matrix"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"ibE9_8EDTMDu","executionInfo":{"status":"ok","timestamp":1626215569148,"user_tz":420,"elapsed":4,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Define Different Models with Embeddings # \n","\n","CLASSIFIERS = [\n","               ['No Embeddings', 'NONE', 50],\n","               ['GLOVE (6B - 50)', './Embeddings/glove.6B.50d.txt', 50],\n","               ['GLOVE (840B - 300)','./Embeddings/glove.840B.300d.txt', 300],\n","               ['Doc2Vec USPTO Patent Embeddings', './Embeddings/W2V Pat Abstracts 50 - AI Bigrams[Lemma].txt', 50],\n","               ['FastText', './Embeddings/FastText.en.300.vec', 300]\n","               ]"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":66,"referenced_widgets":["3e8f651ae9c144849ded02196d727de1","58d6c9b34a044184ab1a43becfdb418a","0f54639e9295433394eccba853256155","fed7798063f243769a7d83448a73c57f","79bd5fef7bff45629805a626bc04ee8f","9fa9f25ecfaa48349fa5387cecde8080","5610bf1a97e6433aa87ad19b481d5093","bc9dd2435d6b48eca811eae8ec25dd27","5bbe24e6da984306866714087af988c3","9a0f3bfef12845fd8cf29a28e49bc134","a531a4f3e85d43249a8bde93793253f1","4232e2b7129d440ba2c3e08e69c1b0e4","f0f80855875b4ab198fa60d0931966a0","c4d9c9d56c9a4bb79a8360459bd0fe48","ae219fc95f6b4711ab73b73616e0e76c","87241e451f3b42c8a8e69fa02416ee6f","0c3d2b30e6d047a9975e8b89a44d84fc","7ef5642ab14b45edae69887b45830869","5184d424658948ec8996d385f06d349d","890580f3713c409796361203cdd0d579","a75bc7f1fb254420b4681e4f45ec907e","66bf4fc292e0443b989c3cf7ad9327b4","b94664e9d02145579be95b2bcd30dee5","cebb89edeae540e79272a4816e228439","ab296e24299840e38e63b2b456486309","d9168727aaad4dc0991bc9ea489174e6","97c3e6663e4d4de0b07f208a6aa33be2","28851d0321114672a69c947555861ddb","35043a05b7304abf95151b7f7f5d63f3","4d1782236f174497b1cd59d85f9c81bc","10fee2087b634b53899ddf0510919f96","8d6ded4fbec241eeab7e423a25f1ce58","d3ec62e1e11b4e5fa47041f1d8e9240f","b0a30546cbe64a85b1ad176937463ed8","7d3442e9b3c5465d809546d2f0f10e7d","ae9d358158554ea1b434e2d3f7109594","414bdc2a145b48b8b2b48da2125b0f76","a4751bc3c2884df782102bac010eb242","c901457a48a94ab59f96f8c5dd8a5089","147748405abb44ec8c11ca0fbbf8d7d5","894fbe5b51d147e09af22d4a22d8ab60","189bf04254b54a2488ffd1cc3b1e3fb3","f8036fd40c9f474180d7246532dd78e3","3767d93432cf477092339dd93c3693d5","530e8f42828443889129923cb1efeb17","c1553306c7334c1a88186f745ae9970f","59da9f4cdf264b09a7c587eef5cc08bf","15c4e7f41b67431fa91814cc400dabe3"]},"id":"8w-zIZKY7ynz","executionInfo":{"status":"ok","timestamp":1626216652024,"user_tz":420,"elapsed":1082879,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}},"outputId":"42710fa6-80d7-4fe2-f7ea-e582319e392c"},"source":["# Define CNN  model Parameters # \n","\n","maxlen = 200\n","batch_size = 50\n","epochs = 20\n","NUM_OF_SPLITS =5\n","\n","\n","# Define arrays in which to store classification outputs # \n","RESULTS = []\n","Classified_Values =[]\n","\n","for params in tqdm_notebook(CLASSIFIERS, \n","                            desc = 'Loop Through Embeddings',\n","                            leave = True): \n","  name = params[0]\n","  path = params[1]\n","  embedding_dim = params[2]\n","\n","\n","  # Load Embedding Matrix # \n","\n","  if path != \"NONE\":\n","    embedding_matrix = create_embedding_matrix(path,tokenizer.word_index, embedding_dim)\n","  else:\n","    embedding_matrix = create_empty_matrix(tokenizer.word_index, embedding_dim)\n","\n","  # Define Lists to Store Data # \n","\n","  y_actual = []\n","  y_predicted = []\n","  id_s = []\n","\n","  for train, test in tqdm_notebook(StratifiedKFold(n_splits= NUM_OF_SPLITS, shuffle=True).split(Abstracts, Classes), \n","                                          desc = 'Cross-Validating',\n","                                          leave = False,\n","                                          total = NUM_OF_SPLITS):\n","\n","      Test_IDs = np.array(IDs)[test].tolist()\n","\n","      X_train = np.array(Abstracts)[train].tolist()\n","      X_test = np.array(Abstracts)[test].tolist()\n","\n","      y_train = np.array(Classes)[train].tolist()\n","      y_test = np.array(Classes)[test].tolist()\n","\n","      # Tokenize using Keras tools - Different from sklearn #\n","\n","      X_train = tokenizer.texts_to_sequences(X_train)\n","      X_test = tokenizer.texts_to_sequences(X_test)\n","\n","      X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\n","      X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)\n","\n","\n","      # Define Model #\n","\n","      model = Sequential()\n","      \n","      if path == \"NONE\":\n","\n","        model.add(layers.Embedding(vocab_size, embedding_dim,\n","                                  #weights=[embedding_matrix],\n","                                  input_length=maxlen,\n","                                  trainable=True))\n","      else: \n","\n","        model.add(layers.Embedding(vocab_size, embedding_dim,\n","                                  weights=[embedding_matrix],\n","                                  input_length=maxlen,\n","                                  trainable=True))\n","\n","      model.add(Dropout(0.2))\n","      model.add(layers.Conv1D(filters=64, kernel_size=2, activation='relu'))\n","      model.add(layers.MaxPooling1D(pool_size = 4))\n","      model.add(layers.Dense(512, activation='sigmoid'))\n","      model.add(layers.LSTM(100))\n","      model.add(layers.Dense(1, activation='sigmoid'))\n","\n","      #model.add(Conv1D(filters=128, kernel_size=5, padding='same', activation='relu'))\n","      #model.add(MaxPooling1D(3))\n","      #model.add(GlobalMaxPooling1D())\n","      #model.add(BatchNormalization())\n","\n","      # Add fully connected layers\n","      #model.add(Dense(50, activation='relu'))\n","      #model.add(Dropout(0.3))\n","      #model.add(Dense(1, activation='sigmoid'))\n","\n","      model.compile(optimizer='adam',\n","                    loss='binary_crossentropy',\n","                    metrics=[tf.metrics.Recall()])\n","      \n","      #model.summary()\n","\n","      #tf.keras.metrics.Recall()\n","\n","      # Run Model #\n","\n","      model.fit(np.array(X_train), \n","                np.array(y_train),  \n","                batch_size=batch_size, \n","                epochs=epochs, \n","                verbose=False,\n","                validation_data = (np.array(X_test), np.array(y_test)))\n","      \n","      #score = model.evaluate(np.array(X_test), \n","      #                       np.array(C_test), \n","      #                       batch_size=batch_size, \n","      #                       verbose=False)\n","\n","      y_pred_p = model.predict(X_test)\n","      y_pred = model.predict_classes(X_test)\n","\n","\n","      # Add to List with Final Results # \n","      y_actual = y_actual + list(y_test)\n","      y_predicted = y_predicted + list(y_pred)\n","      id_s = id_s + list(Test_IDs)\n","\n","\n","\n","  # ---------------------------------------------------------- #\n","  # This runs only after all of the folds have been classified # \n","  # ---------------------------------------------------------- #\n","\n","  # Compute the Share of AI Patents #\n","  Share = np.round(np.mean(y_predicted), 3)\n","\n","  # Calculate Model Performance Metrics #\n","  Accuracy = accuracy_score(y_actual, y_predicted)\n","  ROC = roc_auc_score(y_actual, y_predicted)\n","  Precision = precision_score(y_actual, y_predicted)\n","  Recall = recall_score(y_actual, y_predicted)\n","  F1 = f1_score(y_actual, y_predicted)\n","  CM = confusion_matrix(y_actual, y_predicted)\n","\n","  # Round to 3 Decimal Places # \n","  #FN = np.round(CM[0][0]/CM[0].sum(), 3)\n","  #FP = np.round(CM[0][1]/CM[0].sum(), 3)\n","  #TN = np.round(CM[1][0]/CM[1].sum(), 3)\n","  #TP = np.round(CM[1][1]/CM[1].sum(), 3)\n","\n","  FN = np.round(CM[0][0]/(CM[0][0] + CM[1][0]), 3)\n","  FP = np.round(CM[0][1]/(CM[0][1] + CM[1][1]), 3)\n","  TN = np.round(CM[1][0]/(CM[0][0] + CM[1][0]), 3)\n","  TP = np.round(CM[1][1]/(CM[0][1] + CM[1][1]), 3)\n","  \n","  # Add Classification Performance Metrics to List#\n","  RESULTS.append([name, Share, TP, FN, FP, TN,\n","                                          np.round(Accuracy, 3),\n","                                          np.round(ROC, 3),\n","                                          np.round(Precision, 3),\n","                                          np.round(Recall, 3),\n","                                          np.round(F1, 3)])\n","\n","  # Add Classification Results to List # \n","  Classified_Values.append(list(zip(len(id_s)*[name],id_s, y_actual, y_predicted)))\n"],"execution_count":10,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3e8f651ae9c144849ded02196d727de1","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Loop Through Embeddings', max=5.0, style=ProgressStyle(de…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5bbe24e6da984306866714087af988c3","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Cross-Validating', max=5.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0c3d2b30e6d047a9975e8b89a44d84fc","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Cross-Validating', max=5.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ab296e24299840e38e63b2b456486309","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Cross-Validating', max=5.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d3ec62e1e11b4e5fa47041f1d8e9240f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Cross-Validating', max=5.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"894fbe5b51d147e09af22d4a22d8ab60","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Cross-Validating', max=5.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"TChVBc8Sy4vp","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1626216652665,"user_tz":420,"elapsed":656,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}},"outputId":"08423c09-7a98-4205-8601-f2ef0a532047"},"source":["# Convert List to Dataframe #\n","RESULTS_TABLE = pd.DataFrame(RESULTS, columns = [\"Name\", \"Share\", \"True-Positives\", \n","                                                 \"False-Negatives\", \"False-Positives\", \n","                                                 \"True-Negatives\",\"Accuracy\", \"AUC\", \n","                                                 \"Precision\", \"Recall\", \"F1\"] )\n","\n","RESULTS_TABLE[\"Type\"] = \"CNN\"\n","RESULTS_TABLE = RESULTS_TABLE[[\"Name\", \"Type\", \"Share\", \"True-Positives\", \n","                               \"False-Negatives\", \"False-Positives\", \n","                               \"True-Negatives\",\"Accuracy\", \"AUC\", \n","                               \"Precision\", \"Recall\", \"F1\"]]\n","\n","# Output Results #\n","RESULTS_TABLE.sort_values(\"Accuracy\", ascending = False ).to_csv(\"./Output/Model Performance/CNN Model Classification Performance.csv\")\n","\n","# Display Results -- Out of Sample (Holdout) prediction -- Sorted by Accuracy #\n","RESULTS_TABLE.sort_values(\"Accuracy\", ascending = False )"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Name</th>\n","      <th>Type</th>\n","      <th>Share</th>\n","      <th>True-Positives</th>\n","      <th>False-Negatives</th>\n","      <th>False-Positives</th>\n","      <th>True-Negatives</th>\n","      <th>Accuracy</th>\n","      <th>AUC</th>\n","      <th>Precision</th>\n","      <th>Recall</th>\n","      <th>F1</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>3</th>\n","      <td>Doc2Vec USPTO Patent Embeddings</td>\n","      <td>CNN</td>\n","      <td>0.203</td>\n","      <td>0.842</td>\n","      <td>0.963</td>\n","      <td>0.158</td>\n","      <td>0.037</td>\n","      <td>0.938</td>\n","      <td>0.906</td>\n","      <td>0.842</td>\n","      <td>0.853</td>\n","      <td>0.848</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>GLOVE (6B - 50)</td>\n","      <td>CNN</td>\n","      <td>0.190</td>\n","      <td>0.855</td>\n","      <td>0.953</td>\n","      <td>0.145</td>\n","      <td>0.047</td>\n","      <td>0.934</td>\n","      <td>0.888</td>\n","      <td>0.855</td>\n","      <td>0.810</td>\n","      <td>0.832</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>GLOVE (840B - 300)</td>\n","      <td>CNN</td>\n","      <td>0.194</td>\n","      <td>0.834</td>\n","      <td>0.952</td>\n","      <td>0.166</td>\n","      <td>0.048</td>\n","      <td>0.929</td>\n","      <td>0.884</td>\n","      <td>0.834</td>\n","      <td>0.808</td>\n","      <td>0.821</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>FastText</td>\n","      <td>CNN</td>\n","      <td>0.193</td>\n","      <td>0.811</td>\n","      <td>0.946</td>\n","      <td>0.189</td>\n","      <td>0.054</td>\n","      <td>0.920</td>\n","      <td>0.868</td>\n","      <td>0.811</td>\n","      <td>0.782</td>\n","      <td>0.796</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>No Embeddings</td>\n","      <td>CNN</td>\n","      <td>0.193</td>\n","      <td>0.777</td>\n","      <td>0.938</td>\n","      <td>0.223</td>\n","      <td>0.062</td>\n","      <td>0.907</td>\n","      <td>0.848</td>\n","      <td>0.777</td>\n","      <td>0.749</td>\n","      <td>0.763</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                              Name Type  Share  ...  Precision  Recall     F1\n","3  Doc2Vec USPTO Patent Embeddings  CNN  0.203  ...      0.842   0.853  0.848\n","1                  GLOVE (6B - 50)  CNN  0.190  ...      0.855   0.810  0.832\n","2               GLOVE (840B - 300)  CNN  0.194  ...      0.834   0.808  0.821\n","4                         FastText  CNN  0.193  ...      0.811   0.782  0.796\n","0                    No Embeddings  CNN  0.193  ...      0.777   0.749  0.763\n","\n","[5 rows x 12 columns]"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"code","metadata":{"id":"jWgFlEeOz037","executionInfo":{"status":"ok","timestamp":1626216654116,"user_tz":420,"elapsed":1466,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":["# Output Classification Results for Training Dataset -- PREDICTED VALUES -- Out Of Sample (Holdout) Prediction # \n","\n","for i in range(0,len(Classified_Values), 1):\n","\n","  Temp = pd.DataFrame(  Classified_Values[i],\n","                        columns = ['Model', 'id', 'Actual', 'Predicted'] )\n","  \n","  if i == 0: \n","    name = Temp.head(1)['Model'][0]\n","    Temp = Temp[['id', 'Actual', 'Predicted']]\n","    Temp.columns = ['id', 'Actual', name]\n","    Final = Temp\n","\n","  else: \n","\n","    name = Temp.head(1)['Model'][0]\n","    Temp = Temp[['id', 'Predicted']]\n","    Temp.columns = ['id', name]\n","\n","    Final = Final.merge(Temp, on = ['id'])\n","\n","# Save Data Frame # \n","Final.to_csv(\"./Output/Classification Output/CNN Classification Results.csv\")"],"execution_count":12,"outputs":[]},{"cell_type":"code","metadata":{"id":"JmuJI-uXJNBD","executionInfo":{"status":"ok","timestamp":1626216654121,"user_tz":420,"elapsed":38,"user":{"displayName":"Annoymous Researcher","photoUrl":"","userId":"13222073037946805330"}}},"source":[""],"execution_count":12,"outputs":[]}]}