diff --git a/ammico/__init__.py b/ammico/__init__.py index 23b20d91..64554061 100644 --- a/ammico/__init__.py +++ b/ammico/__init__.py @@ -3,7 +3,7 @@ except ImportError: # Running on pre-3.8 Python; use importlib-metadata package import importlib_metadata as metadata # type: ignore -from ammico.cropposts import crop_media_posts +from ammico.cropposts import crop_media_posts, crop_posts_from_refs from ammico.display import AnalysisExplorer from ammico.faces import EmotionDetector from ammico.multimodal_search import MultimodalSearch @@ -18,6 +18,7 @@ __all__ = [ "crop_media_posts", + "crop_posts_from_refs", "AnalysisExplorer", "EmotionDetector", "MultimodalSearch", diff --git a/ammico/display.py b/ammico/display.py index 387e9269..3034df44 100644 --- a/ammico/display.py +++ b/ammico/display.py @@ -110,6 +110,8 @@ def __init__(self, mydict: dict) -> None: State("left_select_id", "value"), State("Dropdown_select_Detector", "value"), State("setting_Text_analyse_text", "value"), + State("setting_Text_model_names", "value"), + State("setting_Text_revision_numbers", "value"), State("setting_Emotion_emotion_threshold", "value"), State("setting_Emotion_race_threshold", "value"), State("setting_Color_delta_e_method", "value"), @@ -177,6 +179,48 @@ def _create_setting_layout(self): ["Analyse text"], id="setting_Text_analyse_text", ), + html.Div( + [ + html.Div( + "Select models for text_summary, text_sentiment, text_NER or leave blank for default:", + style={ + "height": "30px", + "margin-top": "5px", + }, + ), + dcc.Input( + type="text", + id="setting_Text_model_names", + style={"height": "auto", "margin-bottom": "auto"}, + ), + ], + style={ + "width": "33%", + "display": "inline-block", + "margin-top": "10px", + }, + ), + html.Div( + [ + html.Div( + "Select model revision number for text_summary, text_sentiment, text_NER or leave blank for default:", + style={ + "height": "30px", + "margin-top": "5px", + }, + ), + dcc.Input( + type="text", + id="setting_Text_revision_numbers", + style={"height": "auto", "margin-bottom": "auto"}, + ), + ], + style={ + "width": "33%", + "display": "inline-block", + "margin-top": "10px", + }, + ), ], ), html.Div( @@ -414,7 +458,9 @@ def _right_output_analysis( all_img_options: dict, current_img_value: str, detector_value: str, - settings_text_analyse_text: bool, + settings_text_analyse_text: list, + settings_text_model_names: str, + settings_text_revision_numbers: str, setting_emotion_emotion_threshold: int, setting_emotion_race_threshold: int, setting_color_delta_e_method: str, @@ -448,8 +494,18 @@ def _right_output_analysis( identify_function = identify_dict[detector_value] if detector_value == "TextDetector": + analyse_text = ( + True if settings_text_analyse_text == ["Analyse text"] else False + ) detector_class = identify_function( - image_copy, analyse_text=settings_text_analyse_text + image_copy, + analyse_text=analyse_text, + model_names=[settings_text_model_names] + if (settings_text_model_names is not None) + else None, + revision_numbers=[settings_text_revision_numbers] + if (settings_text_revision_numbers is not None) + else None, ) elif detector_value == "EmotionDetector": detector_class = identify_function( diff --git a/ammico/multimodal_search.py b/ammico/multimodal_search.py index c047edfd..47649f98 100644 --- a/ammico/multimodal_search.py +++ b/ammico/multimodal_search.py @@ -935,7 +935,7 @@ def image_text_match_reordering( return itm_scores2, image_gradcam_with_itm def show_results( - self, query: dict, itm=False, image_gradcam_with_itm=False + self, query: dict, itm: bool = False, image_gradcam_with_itm: dict = {} ) -> None: """ Show results of search. @@ -943,7 +943,7 @@ def show_results( Args: query (dict): query. itm (bool): use itm model. Default: False. - image_gradcam_with_itm (bool): use gradcam. Default: False. + image_gradcam_with_itm (dict): use gradcam. Default: empty. """ if "image" in query.keys(): pic = Image.open(query["image"]).convert("RGB") @@ -972,11 +972,11 @@ def show_results( ): if s[1][current_querry_rank] is None: break - if image_gradcam_with_itm is False: - p1 = Image.open(s[1]["filename"]).convert("RGB") - else: + if bool(image_gradcam_with_itm) is True and itm is True: image = image_gradcam_with_itm[list(query.values())[0]][s[0]] p1 = Image.fromarray(image.astype("uint8"), "RGB") + else: + p1 = Image.open(s[1]["filename"]).convert("RGB") p1.thumbnail((400, 400)) display( "Rank: " diff --git a/ammico/notebooks/colors_analysis.ipynb b/ammico/notebooks/colors_analysis.ipynb index b923e104..f922b435 100644 --- a/ammico/notebooks/colors_analysis.ipynb +++ b/ammico/notebooks/colors_analysis.ipynb @@ -45,9 +45,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ammico\n", - "from ammico import utils as mutils\n", - "from ammico import display as mdisplay\n" + "import ammico" ] }, { @@ -66,7 +64,7 @@ "source": [ "# Here you need to provide the path to your google drive folder\n", "# or local folder containing the images\n", - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")\n" @@ -86,7 +84,7 @@ "metadata": {}, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" + "mydict = ammico.initialize_dict(images)" ] }, { @@ -104,7 +102,7 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", + "analysis_explorer = ammico.AnalysisExplorer(mydict)\n", "analysis_explorer.run_server(port = 8057)" ] }, @@ -140,8 +138,8 @@ "metadata": {}, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { @@ -195,7 +193,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/ammico/notebooks/cropposts.ipynb b/ammico/notebooks/cropposts.ipynb index c45e2d2a..2c2b6101 100644 --- a/ammico/notebooks/cropposts.ipynb +++ b/ammico/notebooks/cropposts.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "b25986d7", "metadata": {}, @@ -10,7 +9,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "c8a5a491", "metadata": {}, @@ -55,8 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ammico.cropposts as crpo\n", - "import ammico.utils as utils\n", + "import ammico\n", "import matplotlib.pyplot as plt\n", "import cv2\n", "import importlib_resources\n", @@ -64,7 +61,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "e7b8127f", "metadata": {}, @@ -81,7 +77,7 @@ "source": [ "# load ref view for cropping the same type social media posts images.\n", "# substitute the below paths for your samples\n", - "path_ref = pkg / \"data\" / \"ref\" / \"ref-00.png\"\n", + "path_ref = \"/content/ref/ref-00.png\"\n", "ref_view = cv2.imread(path_ref)\n", "RGB_ref_view = cv2.cvtColor(ref_view, cv2.COLOR_BGR2RGB)\n", "plt.figure(figsize=(10, 15))\n", @@ -89,7 +85,7 @@ "plt.show()\n", "\n", "path_post = pkg / \"data\" / \"test-crop-image.png\"\n", - "view = cv2.imread(path_post)\n", + "view = cv2.imread(path_post.as_posix())\n", "RGB_view = cv2.cvtColor(view, cv2.COLOR_BGR2RGB)\n", "plt.figure(figsize=(10, 15))\n", "plt.imshow(RGB_view)\n", @@ -97,7 +93,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "49a11f61", "metadata": {}, @@ -114,14 +109,13 @@ "source": [ "# crop a posts from reference view, check the cropping \n", "# this will only plot something if the reference is found on the image\n", - "crop_view = crpo.crop_posts_from_refs(\n", + "crop_view = ammico.crop_posts_from_refs(\n", " [ref_view], view, \n", " plt_match=True, plt_crop=True, plt_image=True,\n", ")" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "1929e549", "metadata": {}, @@ -139,14 +133,14 @@ "outputs": [], "source": [ "\n", - "crop_dir = \"../ammico/data/\"\n", - "ref_dir = \"../ammico/data/ref\"\n", - "save_crop_dir = \"data/crop/\"\n", + "crop_dir = \"/content/drive/MyDrive/misinformation-data/\"\n", + "ref_dir = \"/content/ref/\"\n", + "save_crop_dir = \"/content/drive/MyDrive/misinformation-data/crop/\"\n", "\n", - "files = utils.find_files(path=crop_dir,limit=10,)\n", - "ref_files = utils.find_files(path=ref_dir, limit=100)\n", + "files = ammico.find_files(path=crop_dir,limit=10,)\n", + "ref_files = ammico.find_files(path=ref_dir, limit=100)\n", "\n", - "crpo.crop_media_posts(files, ref_files, save_crop_dir, plt_match=True, plt_crop=False, plt_image=False)\n", + "ammico.crop_media_posts(files, ref_files, save_crop_dir, plt_match=True, plt_crop=False, plt_image=False)\n", "print(\"Batch cropping images done\")" ] }, diff --git a/ammico/notebooks/facial_expressions.ipynb b/ammico/notebooks/facial_expressions.ipynb index b2047282..244ae4db 100644 --- a/ammico/notebooks/facial_expressions.ipynb +++ b/ammico/notebooks/facial_expressions.ipynb @@ -52,9 +52,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ammico\n", - "from ammico import utils as mutils\n", - "from ammico import display as mdisplay" + "import ammico" ] }, { @@ -75,7 +73,7 @@ "source": [ "# Here you need to provide the path to your google drive folder\n", "# or local folder containing the images\n", - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")" @@ -97,7 +95,7 @@ "metadata": {}, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" + "mydict = ammico.initialize_dict(images)" ] }, { @@ -117,7 +115,7 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", + "analysis_explorer = ammico.AnalysisExplorer(mydict)\n", "analysis_explorer.run_server(port = 8050)" ] }, @@ -138,7 +136,7 @@ "outputs": [], "source": [ "for key in mydict.keys():\n", - " mydict[key] = ammico.faces.EmotionDetector(mydict[key]).analyse_image()" + " mydict[key] = ammico.EmotionDetector(mydict[key]).analyse_image()" ] }, { @@ -157,8 +155,8 @@ "metadata": {}, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { @@ -224,7 +222,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.3" }, "vscode": { "interpreter": { diff --git a/ammico/notebooks/get-text-from-image.ipynb b/ammico/notebooks/get-text-from-image.ipynb index 10292300..989f5c1a 100644 --- a/ammico/notebooks/get-text-from-image.ipynb +++ b/ammico/notebooks/get-text-from-image.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "dcaa3da1", "metadata": {}, @@ -57,13 +56,10 @@ "outputs": [], "source": [ "import os\n", - "import ammico\n", - "from ammico import utils as mutils\n", - "from ammico import display as mdisplay" + "import ammico" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "fddba721", "metadata": {}, @@ -80,14 +76,13 @@ "source": [ "# Here you need to provide the path to your google drive folder\n", "# or local folder containing the images\n", - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "3a7dfe11", "metadata": {}, @@ -102,11 +97,10 @@ "metadata": {}, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" + "mydict = ammico.initialize_dict(images)" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "7b8b929f", "metadata": {}, @@ -129,7 +123,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "0891b795-c7fe-454c-a45d-45fadf788142", "metadata": {}, @@ -146,12 +139,11 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", + "analysis_explorer = ammico.AnalysisExplorer(mydict)\n", "analysis_explorer.run_server(port=8054)" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "9c3e72b5-0e57-4019-b45e-3e36a74e7f52", "metadata": {}, @@ -168,13 +160,12 @@ "outputs": [], "source": [ "for key in mydict:\n", - " mydict[key] = ammico.text.TextDetector(\n", + " mydict[key] = ammico.TextDetector(\n", " mydict[key], analyse_text=True\n", " ).analyse_image()" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "3c063eda", "metadata": {}, @@ -190,12 +181,11 @@ "metadata": {}, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "ae182eb7", "metadata": {}, @@ -214,7 +204,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "eedf1e47", "metadata": {}, @@ -234,7 +223,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "4bc8ac0a", "metadata": {}, @@ -244,7 +232,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "4931941b", "metadata": {}, @@ -266,13 +253,12 @@ "outputs": [], "source": [ "# make a list of all the text_english entries per analysed image from the mydict variable as above\n", - "topic_model, topic_df, most_frequent_topics = ammico.text.PostprocessText(\n", + "topic_model, topic_df, most_frequent_topics = ammico.PostprocessText(\n", " mydict=mydict\n", ").analyse_topic()" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "95667342", "metadata": {}, @@ -289,13 +275,12 @@ "outputs": [], "source": [ "input_file_path = \"/content/drive/MyDrive/misinformation-data/data_out.csv\"\n", - "topic_model, topic_df, most_frequent_topics = ammico.text.PostprocessText(\n", + "topic_model, topic_df, most_frequent_topics = ammico.PostprocessText(\n", " use_csv=True, csv_path=input_file_path\n", ").analyse_topic(return_topics=10)" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "0b6ef6d7", "metadata": {}, @@ -315,7 +300,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "b3316770", "metadata": {}, @@ -336,7 +320,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "d10f701e", "metadata": {}, @@ -356,7 +339,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "f4eaf353", "metadata": {}, diff --git a/ammico/notebooks/image_summary.ipynb b/ammico/notebooks/image_summary.ipynb index f200b09d..79ee1ff9 100644 --- a/ammico/notebooks/image_summary.ipynb +++ b/ammico/notebooks/image_summary.ipynb @@ -13,11 +13,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebooks shows how to generate image captions and use the visual question answering with [LAVIS](https://github.com/salesforce/LAVIS). \n", + "This notebooks shows how to generate image captions and use the visual question answering with [LAVIS](https://github.com/salesforce/LAVIS) library. \n", "\n", "The first cell is only run on google colab and installs the [ammico](https://github.com/ssciwr/AMMICO) package.\n", "\n", - "After that, we can import `ammico` and read in the files given a folder path." + "After that, you can import `ammico` and read in the files given a folder path." ] }, { @@ -50,10 +50,7 @@ }, "outputs": [], "source": [ - "import ammico\n", - "from ammico import utils as mutils\n", - "from ammico import display as mdisplay\n", - "import ammico.summary as sm" + "import ammico" ] }, { @@ -66,7 +63,7 @@ "source": [ "# Here you need to provide the path to your google drive folder\n", "# or local folder containing the images\n", - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")" @@ -80,7 +77,7 @@ }, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" + "mydict = ammico.initialize_dict(images)" ] }, { @@ -96,9 +93,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Here you can choose between two models: \"base\" or \"large\". This will generate the caption for each image and directly put the results in a dataframe. This dataframe can be exported as a csv file.\n", + "Here you can choose between two models: `\"base\"` or `\"large\"`. This will generate the caption for each image and directly put the results in your dictionary `mydict`. Then you can transform it into the dataframe and this dataframe can be exported as a .csv file.\n", "\n", - "The results are written into the columns `const_image_summary` - this will always be the same result (as always the same seed will be used). The column `3_non-deterministic summary` displays three different answers generated with different seeds, these are most likely different when you run the analysis again." + "The results are written in the columns: \n", + "- `const_image_summary` - the permanent summaries, which do not change from run to run (analyse_image).\n", + "- `3_non-deterministic summary` displays three different summaries generated with different seeds that change from run to run (analyse_image). \n", + "\n", + "You can also specify what kind of analysis you want to perform with `analysis_type`. `\"summary\"` will generate a summary for all pictures in your dictionary `mydict`, `\"questions\"` will prepare answers to your questions for all pictures, and `\"summary_and_questions\"` will do both. \n", + "If you load the models (`summary_model`, `summary_vis_processors` for `\"summary\"` and `summary_vqa_model`, `summary_vqa_vis_processors`, `summary_vqa_txt_processors` for `\"questions\"`) into memory beforehand and pass them to the function, it can speed up the analysis many times. \n", + "\n" ] }, { @@ -109,9 +112,9 @@ }, "outputs": [], "source": [ - "obj = sm.SummaryDetector(mydict)\n", - "summary_model, summary_vis_processors = obj.load_model(model_type=\"base\")\n", - "# summary_model, summary_vis_processors = mutils.load_model(\"large\")" + "obj = ammico.SummaryDetector(mydict)\n", + "summary_model, summary_vis_processors = obj.load_model(model_type=\"base\") # here we load the base model to the memory. This can dramatically speed up the calculation process then.\n", + "# summary_model, summary_vis_processors = ammico.load_model(\"large\")" ] }, { @@ -123,9 +126,12 @@ "outputs": [], "source": [ "for key in mydict:\n", - " mydict[key] = sm.SummaryDetector(mydict[key]).analyse_image(\n", - " summary_model=summary_model, summary_vis_processors=summary_vis_processors\n", - " )" + " mydict[key] = ammico.SummaryDetector(\n", + " mydict[key], # here we pass the dictionary containing the images\n", + " analysis_type=\"summary\", # here we specify the type of analysis to perform (summary, questions, summary_and_questions)\n", + " summary_model=summary_model, # here we pass the model to use for the analysis\n", + " summary_vis_processors=summary_vis_processors # here we pass the visual processors to use for the analysis\n", + " ).analyse_image()" ] }, { @@ -135,6 +141,8 @@ "tags": [] }, "source": [ + "### Convert to dataframe and write csv\n", + "\n", "Convert the dictionary of dictionarys into a dictionary with lists:" ] }, @@ -146,8 +154,8 @@ }, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { @@ -186,32 +194,6 @@ "df.to_csv(\"/content/drive/MyDrive/misinformation-data/data_out.csv\")" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Manually inspect the summaries\n", - "\n", - "To check the analysis, you can inspect the analyzed elements here. Loading the results takes a moment, so please be patient. If you are sure of what you are doing.\n", - "\n", - "`const_image_summary` - the permanent summarys, which does not change from run to run (analyse_image).\n", - "\n", - "`3_non-deterministic summary` - 3 different summarys examples that change from run to run (analyse_image). " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", - "analysis_explorer.run_server(port=8055)" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -225,7 +207,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Set the list of questions as a list of strings:" + "Set the list of questions as a list of strings `list_of_questions`, load the models to the memory and pass them to the function" ] }, { @@ -242,11 +224,16 @@ ] }, { - "attachments": {}, - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Explore the analysis using the interface:" + "(\n", + " summary_vqa_model, \n", + " summary_vqa_vis_processors, \n", + " summary_vqa_txt_processors \n", + ") = obj.load_vqa_model() # here we load the VQA model to the memory. This can dramatically speed up the calculation process then.\n" ] }, { @@ -255,8 +242,14 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", - "analysis_explorer.run_server(port=8055)" + "for key in mydict:\n", + " mydict[key] = ammico.SummaryDetector(\n", + " mydict[key],\n", + " analysis_type=\"questions\",\n", + " summary_vqa_model=summary_vqa_model,\n", + " summary_vqa_vis_processors=summary_vqa_vis_processors,\n", + " summary_vqa_txt_processors=summary_vqa_txt_processors, \n", + " ).analyse_questions(list_of_questions)" ] }, { @@ -264,8 +257,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Or directly analyze for further processing\n", - "Instead of inspecting each of the images, you can also directly carry out the analysis and export the result into a csv. This may take a while depending on how many images you have loaded." + "Or you can perform two types of analysis at a time `analysis_type=\"summary_and_questions\"`." ] }, { @@ -275,7 +267,15 @@ "outputs": [], "source": [ "for key in mydict:\n", - " mydict[key] = sm.SummaryDetector(mydict[key]).analyse_questions(list_of_questions)" + " mydict[key] = ammico.SummaryDetector(\n", + " mydict[key],\n", + " analysis_type=\"summary_and_questions\",\n", + " summary_model=summary_model, \n", + " summary_vis_processors=summary_vis_processors,\n", + " summary_vqa_model=summary_vqa_model,\n", + " summary_vqa_vis_processors=summary_vqa_vis_processors,\n", + " summary_vqa_txt_processors=summary_vqa_txt_processors, \n", + " ).analyse_questions(list_of_questions)" ] }, { @@ -283,7 +283,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Convert to dataframe and write csv\n", + "### Convert to dataframe and write csv\n", "These steps are required to convert the dictionary of dictionarys into a dictionary with lists, that can be converted into a pandas dataframe and exported to a csv file." ] }, @@ -293,8 +293,8 @@ "metadata": {}, "outputs": [], "source": [ - "outdict2 = mutils.append_data_to_dict(mydict)\n", - "df2 = mutils.dump_df(outdict2)" + "outdict2 = ammico.append_data_to_dict(mydict)\n", + "df2 = ammico.dump_df(outdict2)" ] }, { @@ -315,12 +315,27 @@ "df2.to_csv(\"/content/drive/MyDrive/misinformation-data/data_out2.csv\")" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manually inspect the summaries and visual question answering\n", + "\n", + "To check the analysis, you can inspect the analyzed elements here. Loading the results takes a moment since it loads the big model to memory for every picture, so please be patient. If you are sure of what you are doing. In this widget you can select the picture, the type of analysis and the question.\n" + ] + }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], - "source": [] + "source": [ + "analysis_explorer = ammico.AnalysisExplorer(mydict)\n", + "analysis_explorer.run_server(port=8055)" + ] } ], "metadata": { @@ -339,7 +354,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.3" }, "vscode": { "interpreter": { diff --git a/ammico/notebooks/multimodal_search.ipynb b/ammico/notebooks/multimodal_search.ipynb index 8b3467cc..b2ed911e 100644 --- a/ammico/notebooks/multimodal_search.ipynb +++ b/ammico/notebooks/multimodal_search.ipynb @@ -54,8 +54,7 @@ }, "outputs": [], "source": [ - "import ammico.utils as mutils\n", - "import ammico.multimodal_search as ms" + "import ammico" ] }, { @@ -67,22 +66,12 @@ }, "outputs": [], "source": [ - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "a08bd3a9-e954-4a0e-ad64-6817abd3a25a", - "metadata": {}, - "outputs": [], - "source": [ - "images" - ] - }, { "cell_type": "code", "execution_count": null, @@ -92,17 +81,7 @@ }, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4c091f95-07cf-42c3-82c8-5f3a3c5929f8", - "metadata": {}, - "outputs": [], - "source": [ - "mydict" + "mydict = ammico.initialize_dict(images)" ] }, { @@ -162,17 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "my_obj = ms.MultimodalSearch(mydict)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16603ded-078e-4362-847b-57ad76829327", - "metadata": {}, - "outputs": [], - "source": [ - "my_obj.subdict" + "my_obj = ammico.MultimodalSearch(mydict)" ] }, { @@ -197,16 +166,6 @@ " )" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "f236c3b1-c3a6-471a-9fc5-ef831b675286", - "metadata": {}, - "outputs": [], - "source": [ - "features_image_stacked" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -215,7 +174,7 @@ "source": [ "The images are then processed and stored in a numerical representation, a tensor. These tensors do not change for the same image and same model - so if you run this analysis once, and save the tensors giving a path with the keyword `path_to_save_tensors`, a file with filename `.__saved_features_image.pt` will be placed there.\n", "\n", - "This will save you a lot of time if you want to analyse same images with the same model but different questions. To run using the saved tensors, execute the below code giving the path and name of the tensor file." + "This can save you time if you want to analyse same images with the same model but different questions. To run using the saved tensors, execute the below code giving the path and name of the tensor file." ] }, { @@ -269,10 +228,14 @@ }, "outputs": [], "source": [ + "import importlib_resources # only requare for image query example\n", + "image_example_path = str(importlib_resources.files(\"ammico\") / \"data\" / \"test-crop-image.png\") # creating the path to the image for the image query example\n", + "\n", "search_query3 = [\n", - " {\"text_input\": \"politician press conference\"},\n", + " {\"text_input\": \"politician press conference\"}, \n", " {\"text_input\": \"a world map\"},\n", - " {\"text_input\": \"a dog\"},\n", + " {\"text_input\": \"a dog\"}, # This is how looks text query\n", + " {\"image\": image_example_path}, # This is how looks image query, here `image_example_path` is the path to query image like \"data/test-crop-image.png\"\n", "]" ] }, @@ -329,16 +292,6 @@ "sorted_lists" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "c93d7e88-594d-4095-b5f2-7bf01210dc61", - "metadata": {}, - "outputs": [], - "source": [ - "mydict" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -351,13 +304,11 @@ { "cell_type": "code", "execution_count": null, - "id": "9ad74b21-6187-4a58-9ed8-fd3e80f5a4ed", - "metadata": { - "tags": [] - }, + "id": "c93d7e88-594d-4095-b5f2-7bf01210dc61", + "metadata": {}, "outputs": [], "source": [ - "mydict[\"109237S_spa\"]" + "mydict" ] }, { @@ -379,7 +330,7 @@ "outputs": [], "source": [ "my_obj.show_results(\n", - " search_query3[0],\n", + " search_query3[0], # you can change the index to see the results for other queries\n", ")" ] }, @@ -391,7 +342,7 @@ "source": [ "## Improve the search results\n", "\n", - "For even better results, a slightly different approach has been prepared that can improve search results. It is quite resource-intensive, so it is applied after the main algorithm has found the most relevant images. This approach works only with text queries. Among the parameters you can choose 3 models: `\"blip_base\"`, `\"blip_large\"`, `\"blip2_coco\"`. If you get an `Out of Memory` error, try reducing the batch_size value (minimum = 1), which is the number of images being processed simultaneously. With the parameter `need_grad_cam = True/False` you can enable the calculation of the heat map of each image to be processed. Thus the `image_text_match_reordering` function calculates new similarity values and new ranks for each image. The resulting values are added to the general dictionary." + "For even better results, a slightly different approach has been prepared that can improve search results. It is quite resource-intensive, so it is applied after the main algorithm has found the most relevant images. This approach works only with text queries and it skips image queries. Among the parameters you can choose 3 models: `\"blip_base\"`, `\"blip_large\"`, `\"blip2_coco\"`. If you get an `Out of Memory` error, try reducing the batch_size value (minimum = 1), which is the number of images being processed simultaneously. With the parameter `need_grad_cam = True/False` you can enable the calculation of the heat map of each image to be processed and save them in `image_gradcam_with_itm`. Thus the `image_text_match_reordering()` function calculates new similarity values and new ranks for each image. The resulting values are added to the general dictionary." ] }, { @@ -433,7 +384,7 @@ "id": "9e98c150-5fab-4251-bce7-0d8fc7b385b9", "metadata": {}, "source": [ - "Then using the same output function you can add the `ITM=True` arguments to output the new image order. You can also add the `image_gradcam_with_itm` argument to output the heat maps of the calculated images. " + "Then using the same output function you can add the `itm=True` argument to output the new image order. Remember that for images querys, an error will be thrown with `itm=True` argument. You can also add the `image_gradcam_with_itm` along with `itm=True` argument to output the heat maps of the calculated images." ] }, { @@ -481,8 +432,8 @@ }, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { @@ -528,14 +479,6 @@ "source": [ "df.to_csv(\"/content/drive/MyDrive/misinformation-data/data_out.csv\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b6a79201-7c17-496c-a6a1-b8ecfd3dd1e8", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -554,7 +497,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/ammico/notebooks/objects_expression.ipynb b/ammico/notebooks/objects_expression.ipynb index b3a675bb..aca26719 100644 --- a/ammico/notebooks/objects_expression.ipynb +++ b/ammico/notebooks/objects_expression.ipynb @@ -48,10 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ammico\n", - "from ammico import utils as mutils\n", - "from ammico import display as mdisplay\n", - "import ammico.objects as ob" + "import ammico" ] }, { @@ -70,7 +67,7 @@ "source": [ "# Here you need to provide the path to your google drive folder\n", "# or local folder containing the images\n", - "images = mutils.find_files(\n", + "images = ammico.find_files(\n", " path=\"/content/drive/MyDrive/misinformation-data/\",\n", " limit=10,\n", ")" @@ -82,7 +79,7 @@ "metadata": {}, "outputs": [], "source": [ - "mydict = mutils.initialize_dict(images)" + "mydict = ammico.initialize_dict(images)" ] }, { @@ -101,7 +98,7 @@ "outputs": [], "source": [ "for key in mydict:\n", - " mydict[key] = ob.ObjectDetector(mydict[key]).analyse_image()" + " mydict[key] = ammico.ObjectDetector(mydict[key]).analyse_image()" ] }, { @@ -118,8 +115,8 @@ "metadata": {}, "outputs": [], "source": [ - "outdict = mutils.append_data_to_dict(mydict)\n", - "df = mutils.dump_df(outdict)" + "outdict = ammico.append_data_to_dict(mydict)\n", + "df = ammico.dump_df(outdict)" ] }, { @@ -173,7 +170,7 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_explorer = mdisplay.AnalysisExplorer(mydict)\n", + "analysis_explorer = ammico.AnalysisExplorer(mydict)\n", "analysis_explorer.run_server(port=8056)" ] }, @@ -201,7 +198,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.3" }, "vscode": { "interpreter": { diff --git a/ammico/test/test_display.py b/ammico/test/test_display.py index 3a2253eb..c91d0ed0 100644 --- a/ammico/test/test_display.py +++ b/ammico/test/test_display.py @@ -1,7 +1,29 @@ import json import ammico.display as ammico_display import pytest -import sys + + +@pytest.fixture +def get_options(get_path): + path_img_1 = get_path + "IMG_2809.png" + path_img_2 = get_path + "IMG_2746.png" + + mydict = { + "IMG_2809": {"filename": path_img_1}, + "IMG_2746": {"filename": path_img_2}, + } + + all_options_dict = { + path_img_1: "IMG_2809", + path_img_2: "IMG_2746", + } + return path_img_1, path_img_2, mydict, all_options_dict + + +@pytest.fixture +def get_AE(get_options): + analysis_explorer = ammico_display.AnalysisExplorer(get_options[2]) + return analysis_explorer def test_explore_analysis_faces(get_path): @@ -26,33 +48,20 @@ def test_explore_analysis_objects(get_path): assert sub_dict[key] == outs[key] -@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") -def test_AnalysisExplorer(get_path): - path_img_1 = get_path + "IMG_2809.png" - path_img_2 = get_path + "IMG_2746.png" - - mydict = { - "IMG_2809": {"filename": path_img_1}, - "IMG_2746": {"filename": path_img_2}, - } +def test_AnalysisExplorer(get_AE, get_options): + get_AE.update_picture(get_options[0]) + assert get_AE.update_picture(None) is None - all_options_dict = { - path_img_1: "IMG_2809", - path_img_2: "IMG_2746", - } - - analysis_explorer = ammico_display.AnalysisExplorer(mydict) - - analysis_explorer.update_picture(path_img_1) - - assert analysis_explorer.update_picture(None) is None - analysis_explorer._right_output_analysis( +def test_right_output_analysis_objects(get_AE, get_options): + get_AE._right_output_analysis( 2, - all_options_dict, - path_img_1, + get_options[3], + get_options[0], "ObjectDetector", True, + None, + None, 50, 50, "CIE 1976", @@ -61,13 +70,17 @@ def test_AnalysisExplorer(get_path): "How many people are in the picture?", ) - analysis_explorer._right_output_analysis( + +def test_right_output_analysis_emotions(get_AE, get_options): + get_AE._right_output_analysis( 2, - all_options_dict, - path_img_1, + get_options[3], + get_options[0], "EmotionDetector", True, 50, + None, + None, 50, "CIE 1976", "summary_and_questions", @@ -75,12 +88,16 @@ def test_AnalysisExplorer(get_path): "How many people are in the picture?", ) - analysis_explorer._right_output_analysis( + +def test_right_output_analysis_summary(get_AE, get_options): + get_AE._right_output_analysis( 2, - all_options_dict, - path_img_1, + get_options[3], + get_options[0], "SummaryDetector", True, + None, + None, 50, 50, "CIE 1976", @@ -89,12 +106,16 @@ def test_AnalysisExplorer(get_path): "How many people are in the picture?", ) - analysis_explorer._right_output_analysis( + +def test_right_output_analysis_colors(get_AE, get_options): + get_AE._right_output_analysis( 2, - all_options_dict, - path_img_1, + get_options[3], + get_options[0], "ColorDetector", True, + None, + None, 50, 50, "CIE 1976", @@ -102,6 +123,5 @@ def test_AnalysisExplorer(get_path): "base", "How many people are in the picture?", ) - with pytest.raises(EnvironmentError): - analysis_explorer.run_server(port=8050) + get_AE.run_server(port=8050) diff --git a/pyproject.toml b/pyproject.toml index 68f69872..be88b5d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools==61", + "setuptools>=61", ] build-backend = "setuptools.build_meta" @@ -11,7 +11,6 @@ description = "AI Media and Misinformation Content Analysis Tool" readme = "README.md" maintainers = [ { name = "Inga Ulusoy", email = "ssc@iwr.uni-heidelberg.de" }, - { name = "Dominic Kempf", email = "ssc@iwr.uni-heidelberg.de" }, { name = "Petr Andriushchenko", email = "ssc@iwr.uni-heidelberg.de" }, ] requires-python = ">=3.8"