software_papers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "software_papers.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOhMD02AnU20WaeWilNtzdV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/petermr/docanalysis/blob/main/software_papers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Building a corpus"
      ],
      "metadata": {
        "id": "EHpxWSHWUl9D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pygetpapers"
      ],
      "metadata": {
        "id": "axg5-Nu7vSFy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f1b1b9c2-7826-4822-d485-12bf933d30f4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pygetpapers in /usr/local/lib/python3.7/dist-packages (1.2.3)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (4.2.6)\n",
            "Requirement already satisfied: habanero in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (1.2.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (2.23.0)\n",
            "Requirement already satisfied: xmltodict in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (0.13.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (4.64.0)\n",
            "Requirement already satisfied: dict2xml in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (1.7.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (1.3.5)\n",
            "Requirement already satisfied: configargparse in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (1.5.3)\n",
            "Requirement already satisfied: arxiv in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (1.4.2)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.7/dist-packages (from pygetpapers) (15.0.1)\n",
            "Requirement already satisfied: feedparser in /usr/local/lib/python3.7/dist-packages (from arxiv->pygetpapers) (6.0.10)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.7/dist-packages (from coloredlogs->pygetpapers) (10.0)\n",
            "Requirement already satisfied: sgmllib3k in /usr/local/lib/python3.7/dist-packages (from feedparser->arxiv->pygetpapers) (1.0.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers) (2022.5.18.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers) (1.24.3)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas->pygetpapers) (1.21.6)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->pygetpapers) (2022.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->pygetpapers) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->pygetpapers) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sparqlwrapper\n",
        "# https://rdflib.github.io/sparqlwrapper/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E6-r7IkNM5pt",
        "outputId": "45d7c5ed-2b48-4a67-bf36-ef8777fc0e85"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: sparqlwrapper in /usr/local/lib/python3.7/dist-packages (2.0.0)\n",
            "Requirement already satisfied: rdflib>=6.1.1 in /usr/local/lib/python3.7/dist-packages (from sparqlwrapper) (6.1.1)\n",
            "Requirement already satisfied: isodate in /usr/local/lib/python3.7/dist-packages (from rdflib>=6.1.1->sparqlwrapper) (0.6.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from rdflib>=6.1.1->sparqlwrapper) (57.4.0)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.7/dist-packages (from rdflib>=6.1.1->sparqlwrapper) (3.0.9)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from rdflib>=6.1.1->sparqlwrapper) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->rdflib>=6.1.1->sparqlwrapper) (3.8.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->rdflib>=6.1.1->sparqlwrapper) (4.2.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from isodate->rdflib>=6.1.1->sparqlwrapper) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "9r1a0JkXunR8"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "from SPARQLWrapper import SPARQLWrapper, JSON\n",
        "import pandas as pd\n",
        "from pprint import pprint\n",
        "from pygetpapers import Pygetpapers\n",
        "from lxml import etree"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "endpoint_url = \"https://query.wikidata.org/sparql\"\n",
        "\n",
        "query = \"\"\"SELECT ?item ?itemLabel ?prop ?propLabel ?PMCID  WHERE {\n",
        "  ?item wdt:P31 wd:Q13442814.\n",
        "  ?item wdt:P932 ?PMCID.\n",
        "  {?item ?prop wd:Q513297.}\n",
        "  UNION \n",
        "  {?item ?prop wd:Q181596.}\n",
        "  UNION\n",
        "  {?item ?prop wd:Q3699942.}\n",
        "  \n",
        "  \n",
        "  SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". }\n",
        "}\"\"\"\n",
        "\n",
        "\n",
        "def get_results(endpoint_url, query):\n",
        "    user_agent = \"WDQS-example Python/%s.%s\" % (sys.version_info[0], sys.version_info[1])\n",
        "    # TODO adjust user agent; see https://w.wiki/CX6\n",
        "    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)\n",
        "    sparql.setQuery(query)\n",
        "    sparql.setReturnFormat(JSON)\n",
        "    return sparql.query().convert()\n",
        "\n",
        "\n",
        "results = get_results(endpoint_url, query)\n",
        "\n",
        "list_of_result_bindings = (results['results']['bindings'])\n",
        "\n",
        "list_of_PMCID = []\n",
        "for result in list_of_result_bindings:\n",
        "    list_of_PMCID.append(result[\"PMCID\"][\"value\"])\n",
        "print(list_of_PMCID)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0MRrkEp_Mdi2",
        "outputId": "543921ce-bc7c-4c2b-dd2e-0ab8c5b02535"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['3651407', '3651407', '403769', '403769', '2666812', '2666812', '2844990', '2844990', '2638934', '2638934', '4752027', '4752027', '3855388', '3855388', '4174321', '4174321', '3685583', '3685583', '3031041', '3031041', '3649846', '3649846', '4168754', '4168754', '3835703', '3835703', '3530913', '3530913', '3521214', '3521214', '4184317', '4184317', '4426829', '4426829', '4535771', '4535771', '4909101', '4909101', '3722524', '3722524', '2375448', '2375448', '2572702', '2572702', '2693143', '2693143', '2752617', '2752617', '3009535', '3009535', '2796819', '2796819', '2910028', '2910028', '2958747', '2958747', '2971582', '2971582', '2982160', '2982160', '5472867', '5472867', '3065683', '3065683', '3190406', '3190406', '3228863', '3228863', '3262844', '3262844', '4141640', '4141640', '2935447', '2935447', '3307116', '3307116', '3514294', '3514294', '4183485', '4183485', '3430699', '3430699', '3599859', '3599859', '3582273', '3582273', '3646686', '3646686', '4271147', '4271147', '4277354', '4277354', '3750528', '3750528', '3083346', '5100142', '5100142', '5978649', '5978649', '7422831', '3083346', '4287948', '4287948', '3967109', '3967109', '4236746', '4236746', '4264639', '4264639', '4410647', '4410647', '4499222', '4499222', '4708103', '4708103', '3307106', '3307106', '4642660', '4642660', '4670004', '4670004', '4743236', '4743236', '4826505', '4826505', '3413384', '3413384', '5266326', '5266326', '5260057', '5260057', '5333177', '5333177', '4760270', '4760270', '4816032', '4816032', '3654709', '3654709', '5089131', '5089131', '5370611', '5370611', '5105883', '5105883', '5082607', '5082607', '3962008', '3962008', '4670012', '4670012', '4224201', '4224201', '4111116', '4111116', '5618529', '5618529', '4168749', '4168749', '2971575', '2971575', '2427162', '2427162', '5105880', '5105880', '5167065', '5167065', '4411368', '4411368', '4706054', '4706054', '4642848', '4642848', '4423409', '4423409', '3888149', '3888149', '4103489', '4103489', '4197741', '4197741', '4103495', '4103495', '4653389', '4653389', '4288400', '4288400', '4380025', '4380025', '4176421', '4176421', '2530886', '2530886', '5624584', '5624584', '5638226', '6628208', '6628208', '6974772', '6974772', '1008230', '226690', '226690', '2376761', '3659198', '3659198', '5588246', '5588246', '4828368', '4828368', '5127930', '4497953', '4497953', '5996309', '5996309', '6203375', '6203375', '6392199', '6392199', '6306265', '6306265', '6345265', '6396093', '7718173', '7718173', '5638226', '4133763', '4133763', '4207244', '4207244', '5726607', '5726607', '5860209', '5860209', '6042825', '6042825', '5998007', '5998007', '6013757', '6013757', '6013758', '6013758', '6039939', '6039939', '6051191', '6051191', '6039917', '6039917', '6053696', '6053696', '6800166', '6800166', '6607905', '6607905', '6492420', '6821346', '6821346', '6880260', '6880260', '6717989', '6717989', '6707396', '6707396', '7194485', '7194485', '7584444', '7584444']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "set_of_PMCID = set(list_of_PMCID)\n",
        "list_of_unique_PMCID = list(set_of_PMCID)\n",
        "PMCIDs = []\n",
        "for id in list_of_unique_PMCID:\n",
        "  PMCIDs.append( \"\".join([\"PMC\", str(id)]))\n",
        "\n",
        "print(PMCIDs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GklyO_uuu17U",
        "outputId": "cfe89fff-8c3f-40ec-92c7-7f6eb142bc1c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['PMC6717989', 'PMC6880260', 'PMC5089131', 'PMC403769', 'PMC6628208', 'PMC4411368', 'PMC4236746', 'PMC3888149', 'PMC4168749', 'PMC5167065', 'PMC3722524', 'PMC5472867', 'PMC2530886', 'PMC3307116', 'PMC226690', 'PMC4184317', 'PMC4287948', 'PMC6039917', 'PMC5260057', 'PMC4499222', 'PMC4670012', 'PMC3262844', 'PMC4288400', 'PMC2971582', 'PMC4224201', 'PMC4826505', 'PMC6203375', 'PMC6392199', 'PMC2844990', 'PMC2638934', 'PMC6492420', 'PMC4642660', 'PMC4708103', 'PMC7194485', 'PMC2796819', 'PMC3646686', 'PMC3521214', 'PMC4743236', 'PMC4380025', 'PMC4410647', 'PMC5624584', 'PMC3599859', 'PMC4816032', 'PMC3228863', 'PMC6800166', 'PMC1008230', 'PMC3031041', 'PMC4168754', 'PMC3065683', 'PMC2693143', 'PMC3685583', 'PMC7718173', 'PMC7584444', 'PMC3083346', 'PMC2910028', 'PMC3307106', 'PMC6051191', 'PMC3530913', 'PMC2572702', 'PMC4426829', 'PMC5998007', 'PMC4111116', 'PMC4752027', 'PMC5588246', 'PMC4828368', 'PMC3514294', 'PMC4271147', 'PMC2427162', 'PMC5638226', 'PMC5978649', 'PMC4497953', 'PMC6306265', 'PMC3962008', 'PMC6042825', 'PMC4642848', 'PMC5860209', 'PMC5726607', 'PMC6396093', 'PMC4207244', 'PMC5370611', 'PMC5618529', 'PMC2375448', 'PMC2958747', 'PMC3967109', 'PMC6345265', 'PMC4103489', 'PMC3582273', 'PMC5105880', 'PMC2982160', 'PMC5082607', 'PMC4103495', 'PMC3413384', 'PMC4760270', 'PMC3430699', 'PMC3651407', 'PMC3659198', 'PMC7422831', 'PMC5266326', 'PMC5100142', 'PMC3649846', 'PMC6707396', 'PMC2752617', 'PMC4706054', 'PMC2971575', 'PMC6607905', 'PMC4183485', 'PMC6013758', 'PMC5996309', 'PMC6013757', 'PMC6039939', 'PMC5105883', 'PMC4277354', 'PMC4176421', 'PMC3855388', 'PMC4174321', 'PMC6821346', 'PMC3190406', 'PMC3009535', 'PMC2935447', 'PMC4141640', 'PMC4653389', 'PMC4670004', 'PMC4197741', 'PMC3654709', 'PMC4909101', 'PMC4133763', 'PMC6053696', 'PMC2376761', 'PMC3750528', 'PMC4423409', 'PMC5333177', 'PMC2666812', 'PMC3835703', 'PMC4535771', 'PMC6974772', 'PMC5127930', 'PMC4264639']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query = \" OR \".join(PMCIDs)"
      ],
      "metadata": {
        "id": "Rr9LoTWJMR-n"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pygetpapers_call = Pygetpapers ()\n",
        "pygetpapers_call.run_command(query=query, limit = 300, output = \"software_related_papers\", xml=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BBJFph6QStcm",
        "outputId": "4befc268-cdef-4d87-d644-df5ab8f6b2ec"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO: Total Hits are 138\n",
            "WARNING: Could not find more papers\n",
            "138it [00:00, 127492.06it/s]\n",
            "INFO: Saving XML files to /content/software_related_papers/*/fulltext.xml\n",
            "100%|██████████| 138/138 [00:36<00:00,  3.82it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating a software dictionary"
      ],
      "metadata": {
        "id": "c0OCoUFxUzC6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "endpoint_url = \"https://query.wikidata.org/sparql\"\n",
        "\n",
        "query = \"\"\"#title: Software used in publications\n",
        "SELECT\n",
        "  DISTINCT \n",
        "    ?softwareLabel\n",
        "    ?software\n",
        "\n",
        "WHERE {\n",
        "  {?software wdt:P277 [] .}\n",
        "  UNION\n",
        "  {?software wdt:P306 [] .}\n",
        "  UNION\n",
        "  {?software wdt:P1324 [] .}\n",
        "  ?paper (wdt:P921|wdt:P4510) ?software .\n",
        "  ?paper wdt:P932 [].\n",
        "  SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". }\n",
        "}\n",
        "ORDER BY ASC(?softwareLabel)\"\"\"\n",
        "\n",
        "\n",
        "def get_results(endpoint_url, query):\n",
        "    user_agent = \"WDQS-example Python/%s.%s\" % (sys.version_info[0], sys.version_info[1])\n",
        "    # TODO adjust user agent; see https://w.wiki/CX6\n",
        "    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)\n",
        "    sparql.setQuery(query)\n",
        "    sparql.setReturnFormat(JSON)\n",
        "    return sparql.query().convert()\n",
        "\n",
        "\n",
        "results = get_results(endpoint_url, query)\n",
        "\n"
      ],
      "metadata": {
        "id": "xLiivP8nU2dw"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list_of_binding_results= results[\"results\"][\"bindings\"]\n"
      ],
      "metadata": {
        "id": "-HAV9RWeVjP-"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dictionary_element=  etree.Element(\"software_dictionary\")\n",
        "dictionary_element.attrib['title']=\"software_dictionary\"\n",
        "for result in list_of_binding_results:\n",
        "    try:\n",
        "      entry_element=etree.SubElement(dictionary_element,\"entry\")\n",
        "      entry_element.attrib['term']=result['softwareLabel']['value']\n",
        "      entry_element.attrib['WikdiataURL']=result['software']['value']\n",
        "    except Exception as e:\n",
        "      (f\"Couldn't add {result['softwareLabel']['value']} to amidict\")\n"
      ],
      "metadata": {
        "id": "IycP0X-SWWt6"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"software_dictionary\",mode='w', encoding='utf-8') as f:\n",
        "    f.write(etree.tostring(dictionary_element, pretty_print=True).decode('utf-8'))"
      ],
      "metadata": {
        "id": "Ftz0v5eQXrJO"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/petermr/docanalysis.git#egg=docanalysis"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBkiCxN0NX6c",
        "outputId": "4b880f31-0a2c-4cc4-c636-038740fd9a34"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting docanalysis\n",
            "  Cloning https://github.com/petermr/docanalysis.git to /tmp/pip-install-q05rshad/docanalysis_21e2c5e666714de4800f446899e1e2cc\n",
            "  Running command git clone -q https://github.com/petermr/docanalysis.git /tmp/pip-install-q05rshad/docanalysis_21e2c5e666714de4800f446899e1e2cc\n",
            "Requirement already satisfied: scispacy in /usr/local/lib/python3.7/dist-packages (from docanalysis) (0.4.0)\n",
            "Requirement already satisfied: beautifulsoup4==4.10.0 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (4.10.0)\n",
            "Requirement already satisfied: braceexpand==0.1.7 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (0.1.7)\n",
            "Requirement already satisfied: coloredlogs==15.0.1 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (15.0.1)\n",
            "Requirement already satisfied: ConfigArgParse==1.5.3 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (1.5.3)\n",
            "Requirement already satisfied: lxml==4.7.1 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (4.7.1)\n",
            "Requirement already satisfied: nltk==3.6.7 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (3.6.7)\n",
            "Requirement already satisfied: pandas==1.3.4 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (1.3.4)\n",
            "Requirement already satisfied: py4ami==0.0.16 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (0.0.16)\n",
            "Requirement already satisfied: pygetpapers==1.1.2 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (1.1.2)\n",
            "Requirement already satisfied: pytest==6.2.5 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (6.2.5)\n",
            "Requirement already satisfied: setuptools==60.3.1 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (60.3.1)\n",
            "Requirement already satisfied: spacy==3.0.7 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (3.0.7)\n",
            "Requirement already satisfied: tkinterweb==3.10.7 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (3.10.7)\n",
            "Requirement already satisfied: tqdm==4.62.3 in /usr/local/lib/python3.7/dist-packages (from docanalysis) (4.62.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.7/dist-packages (from beautifulsoup4==4.10.0->docanalysis) (2.3.2.post1)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.7/dist-packages (from coloredlogs==15.0.1->docanalysis) (10.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from nltk==3.6.7->docanalysis) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from nltk==3.6.7->docanalysis) (7.1.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.7/dist-packages (from nltk==3.6.7->docanalysis) (2022.6.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.4->docanalysis) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.4->docanalysis) (2022.1)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.4->docanalysis) (1.21.6)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (7.1.2)\n",
            "Requirement already satisfied: pdfminer3 in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (2018.12.3.0)\n",
            "Requirement already satisfied: SPARQLWrapper in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (2.0.0)\n",
            "Requirement already satisfied: PyPDF2 in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (2.1.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (3.2.2)\n",
            "Requirement already satisfied: python-rake in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (1.5.0)\n",
            "Requirement already satisfied: tkinterhtml in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (0.7)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from py4ami==0.0.16->docanalysis) (5.4.8)\n",
            "Requirement already satisfied: habanero in /usr/local/lib/python3.7/dist-packages (from pygetpapers==1.1.2->docanalysis) (1.2.2)\n",
            "Requirement already satisfied: xmltodict in /usr/local/lib/python3.7/dist-packages (from pygetpapers==1.1.2->docanalysis) (0.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from pygetpapers==1.1.2->docanalysis) (2.23.0)\n",
            "Requirement already satisfied: dict2xml in /usr/local/lib/python3.7/dist-packages (from pygetpapers==1.1.2->docanalysis) (1.7.1)\n",
            "Requirement already satisfied: arxiv in /usr/local/lib/python3.7/dist-packages (from pygetpapers==1.1.2->docanalysis) (1.4.2)\n",
            "Requirement already satisfied: importlib-metadata>=0.12 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (4.11.4)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (1.1.1)\n",
            "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (1.0.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (0.10.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (21.3)\n",
            "Requirement already satisfied: py>=1.8.2 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (1.11.0)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.7/dist-packages (from pytest==6.2.5->docanalysis) (21.4.0)\n",
            "Requirement already satisfied: nmslib>=1.7.3.6 in /usr/local/lib/python3.7/dist-packages (from scispacy->docanalysis) (2.1.1)\n",
            "Requirement already satisfied: pysbd in /usr/local/lib/python3.7/dist-packages (from scispacy->docanalysis) (0.3.4)\n",
            "Requirement already satisfied: conllu in /usr/local/lib/python3.7/dist-packages (from scispacy->docanalysis) (4.4.2)\n",
            "Requirement already satisfied: scikit-learn>=0.20.3 in /usr/local/lib/python3.7/dist-packages (from scispacy->docanalysis) (1.0.2)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.5 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.3 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (8.0.17)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (3.10.0.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (2.0.6)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (1.0.7)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.4 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (2.0.7)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (0.4.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (1.8.2)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (0.6.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (2.11.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (0.9.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (3.0.6)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (2.4.3)\n",
            "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0.7->docanalysis) (0.3.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.4->spacy==3.0.7->docanalysis) (3.8.0)\n",
            "Requirement already satisfied: pybind11<2.6.2 in /usr/local/lib/python3.7/dist-packages (from nmslib>=1.7.3.6->scispacy->docanalysis) (2.6.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->pytest==6.2.5->docanalysis) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy==3.0.7->docanalysis) (5.2.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas==1.3.4->docanalysis) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers==1.1.2->docanalysis) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers==1.1.2->docanalysis) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers==1.1.2->docanalysis) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->pygetpapers==1.1.2->docanalysis) (2022.5.18.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.3->scispacy->docanalysis) (3.1.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.3->scispacy->docanalysis) (1.4.1)\n",
            "Requirement already satisfied: feedparser in /usr/local/lib/python3.7/dist-packages (from arxiv->pygetpapers==1.1.2->docanalysis) (6.0.10)\n",
            "Requirement already satisfied: sgmllib3k in /usr/local/lib/python3.7/dist-packages (from feedparser->arxiv->pygetpapers==1.1.2->docanalysis) (1.0.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy==3.0.7->docanalysis) (2.0.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->py4ami==0.0.16->docanalysis) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->py4ami==0.0.16->docanalysis) (1.4.2)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.7/dist-packages (from pdfminer3->py4ami==0.0.16->docanalysis) (2.4.0)\n",
            "Requirement already satisfied: pycryptodome in /usr/local/lib/python3.7/dist-packages (from pdfminer3->py4ami==0.0.16->docanalysis) (3.14.1)\n",
            "Requirement already satisfied: rdflib>=6.1.1 in /usr/local/lib/python3.7/dist-packages (from SPARQLWrapper->py4ami==0.0.16->docanalysis) (6.1.1)\n",
            "Requirement already satisfied: isodate in /usr/local/lib/python3.7/dist-packages (from rdflib>=6.1.1->SPARQLWrapper->py4ami==0.0.16->docanalysis) (0.6.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m nltk.downloader stopwords\n",
        "!python -m nltk.downloader punkt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VcXm5rGsOnJN",
        "outputId": "eaca9635-64f2-4fbf-b4d9-bb91e5d20a20"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/lib/python3.7/runpy.py:125: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n",
            "  warn(RuntimeWarning(msg))\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "/usr/lib/python3.7/runpy.py:125: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n",
            "  warn(RuntimeWarning(msg))\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!docanalysis --help"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b_XTOw7zN7CA",
        "outputId": "5bce00b8-ef8b-447c-d0a8-8da2f3e00f2a"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/_distutils_hack/__init__.py:36: UserWarning: Setuptools is replacing distutils.\n",
            "  warnings.warn(\"Setuptools is replacing distutils.\")\n",
            "usage: docanalysis [-h] [--run_pygetpapers] [--make_section] [-q QUERY]\n",
            "                   [-k HITS] [--project_name PROJECT_NAME] [-d DICTIONARY]\n",
            "                   [-o OUTPUT] [--make_ami_dict MAKE_AMI_DICT]\n",
            "                   [--search_section [SEARCH_SECTION [SEARCH_SECTION ...]]]\n",
            "                   [--entities [ENTITIES [ENTITIES ...]]]\n",
            "                   [--spacy_model SPACY_MODEL] [--html HTML]\n",
            "                   [--synonyms SYNONYMS] [-l LOGLEVEL] [-f LOGFILE]\n",
            "\n",
            "Welcome to docanalysis version 0.1.0. -h or --help for help\n",
            "\n",
            "optional arguments:\n",
            "  -h, --help            show this help message and exit\n",
            "  --run_pygetpapers     downloads papers from EuropePMC via pygetpapers\n",
            "  --make_section        makes sections\n",
            "  -q QUERY, --query QUERY\n",
            "                        provide query to pygetpapers\n",
            "  -k HITS, --hits HITS  specify number of papers to download from pygetpapers\n",
            "  --project_name PROJECT_NAME\n",
            "                        provide CProject directory name\n",
            "  -d DICTIONARY, --dictionary DICTIONARY\n",
            "                        provide ami dictionary to annotate sentences or\n",
            "                        support supervised entity extraction\n",
            "  -o OUTPUT, --output OUTPUT\n",
            "                        outputs csv file\n",
            "  --make_ami_dict MAKE_AMI_DICT\n",
            "                        provide title for ami-dict. Makes ami-dict of all\n",
            "                        extracted entities\n",
            "  --search_section [SEARCH_SECTION [SEARCH_SECTION ...]]\n",
            "                        provide section(s) to annotate. Choose from: ALL, ACK,\n",
            "                        AFF, AUT, CON, DIS, ETH, FIG, INT, KEY, MET, RES, TAB,\n",
            "                        TIL. Defaults to ALL\n",
            "  --entities [ENTITIES [ENTITIES ...]]\n",
            "                        provide entities to extract. Default(ALL). Choose from\n",
            "                        SpaCy: CARDINAL, DATE, EVENT, FAC, GPE, LANGUAGE, LAW,\n",
            "                        LOC, MONEY, NORP, ORDINAL, ORG, PERCENT, PERSON,\n",
            "                        PRODUCT, QUANTITY, TIME, WORK_OF_ART; SciSpaCy:\n",
            "                        CHEMICAL, DISEASE\n",
            "  --spacy_model SPACY_MODEL\n",
            "                        optional. Choose between spacy or scispacy models.\n",
            "                        Defaults to spacy\n",
            "  --html HTML           saves output in html format to given path\n",
            "  --synonyms SYNONYMS   searches the corpus/sections with synonymns from ami-\n",
            "                        dict\n",
            "  -l LOGLEVEL, --loglevel LOGLEVEL\n",
            "                        provide logging level. Example --log warning\n",
            "                        <<info,warning,debug,error,critical>>, default='info'\n",
            "  -f LOGFILE, --logfile LOGFILE\n",
            "                        saves log to specified file in output directory as\n",
            "                        well as printing to terminal\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!docanalysis --make_section --project_name /content/software_related_papers --dictionary /content/software_related_papers/software_dictionary --output software.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cXox79gMOKF-",
        "outputId": "ebffae9b-d771-40a2-f08c-ce6190b19263"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/_distutils_hack/__init__.py:36: UserWarning: Setuptools is replacing distutils.\n",
            "  warnings.warn(\"Setuptools is replacing distutils.\")\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC4174321/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4535771/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m dict_keys: dict_keys(['abstract', 'acknowledge', 'affiliation', 'author', 'conclusion', 'discussion', 'ethics', 'fig_caption', 'front', 'introduction', 'jrnl_title', 'keyword', 'method', 'octree', 'pdfimage', 'pub_date', 'publisher', 'reference', 'results_discuss', 'search_results', 'sections', 'svg', 'table', 'title'])\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mloading templates.json\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4535771/fulltext.xml /content/software_related_papers/PMC4535771/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3430699/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3430699/fulltext.xml /content/software_related_papers/PMC3430699/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3855388/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3855388/fulltext.xml /content/software_related_papers/PMC3855388/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3262844/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3262844/fulltext.xml /content/software_related_papers/PMC3262844/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4743236/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4743236/fulltext.xml /content/software_related_papers/PMC4743236/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC7584444/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC7584444/fulltext.xml /content/software_related_papers/PMC7584444/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC7718173/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC7718173/fulltext.xml /content/software_related_papers/PMC7718173/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4752027/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4752027/fulltext.xml /content/software_related_papers/PMC4752027/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5624584/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5624584/fulltext.xml /content/software_related_papers/PMC5624584/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6396093/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6396093/fulltext.xml /content/software_related_papers/PMC6396093/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC4271147/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4670004/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4670004/fulltext.xml /content/software_related_papers/PMC4670004/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC4287948/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5105883/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5105883/fulltext.xml /content/software_related_papers/PMC5105883/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC4410647/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4111116/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4111116/fulltext.xml /content/software_related_papers/PMC4111116/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC4497953/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4708103/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4708103/fulltext.xml /content/software_related_papers/PMC4708103/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3190406/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3190406/fulltext.xml /content/software_related_papers/PMC3190406/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC1008230/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4653389/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4653389/fulltext.xml /content/software_related_papers/PMC4653389/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC6607905/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4828368/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4828368/fulltext.xml /content/software_related_papers/PMC4828368/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5100142/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5100142/fulltext.xml /content/software_related_papers/PMC5100142/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5588246/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5588246/fulltext.xml /content/software_related_papers/PMC5588246/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6707396/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6707396/fulltext.xml /content/software_related_papers/PMC6707396/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC8243012/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC8243012/fulltext.xml /content/software_related_papers/PMC8243012/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3514294/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3514294/fulltext.xml /content/software_related_papers/PMC3514294/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2982160/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2982160/fulltext.xml /content/software_related_papers/PMC2982160/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC7422831/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC7422831/fulltext.xml /content/software_related_papers/PMC7422831/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6974772/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6974772/fulltext.xml /content/software_related_papers/PMC6974772/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4277354/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4277354/fulltext.xml /content/software_related_papers/PMC4277354/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC6800166/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3031041/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3031041/fulltext.xml /content/software_related_papers/PMC3031041/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2572702/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2572702/fulltext.xml /content/software_related_papers/PMC2572702/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5333177/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5333177/fulltext.xml /content/software_related_papers/PMC5333177/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4133763/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4133763/fulltext.xml /content/software_related_papers/PMC4133763/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2971575/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2971575/fulltext.xml /content/software_related_papers/PMC2971575/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4168754/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4168754/fulltext.xml /content/software_related_papers/PMC4168754/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4183485/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4183485/fulltext.xml /content/software_related_papers/PMC4183485/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4184317/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4184317/fulltext.xml /content/software_related_papers/PMC4184317/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5618529/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5618529/fulltext.xml /content/software_related_papers/PMC5618529/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5998007/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5998007/fulltext.xml /content/software_related_papers/PMC5998007/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4207244/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4207244/fulltext.xml /content/software_related_papers/PMC4207244/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4423409/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4423409/fulltext.xml /content/software_related_papers/PMC4423409/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5105880/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5105880/fulltext.xml /content/software_related_papers/PMC5105880/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6203375/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6203375/fulltext.xml /content/software_related_papers/PMC6203375/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2935447/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2935447/fulltext.xml /content/software_related_papers/PMC2935447/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6039917/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6039917/fulltext.xml /content/software_related_papers/PMC6039917/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5860209/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5860209/fulltext.xml /content/software_related_papers/PMC5860209/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4499222/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4499222/fulltext.xml /content/software_related_papers/PMC4499222/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6039939/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6039939/fulltext.xml /content/software_related_papers/PMC6039939/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5370611/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5370611/fulltext.xml /content/software_related_papers/PMC5370611/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4103495/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4103495/fulltext.xml /content/software_related_papers/PMC4103495/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5089131/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5089131/fulltext.xml /content/software_related_papers/PMC5089131/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3722524/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3722524/fulltext.xml /content/software_related_papers/PMC3722524/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4288400/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4288400/fulltext.xml /content/software_related_papers/PMC4288400/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3083346/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3083346/fulltext.xml /content/software_related_papers/PMC3083346/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4816032/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4816032/fulltext.xml /content/software_related_papers/PMC4816032/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2375448/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2375448/fulltext.xml /content/software_related_papers/PMC2375448/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4103489/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4103489/fulltext.xml /content/software_related_papers/PMC4103489/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6053696/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6053696/fulltext.xml /content/software_related_papers/PMC6053696/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4642660/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4642660/fulltext.xml /content/software_related_papers/PMC4642660/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6013757/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6013757/fulltext.xml /content/software_related_papers/PMC6013757/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC403769/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2693143/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2693143/fulltext.xml /content/software_related_papers/PMC2693143/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC2844990/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5996309/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5996309/fulltext.xml /content/software_related_papers/PMC5996309/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4909101/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4909101/fulltext.xml /content/software_related_papers/PMC4909101/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC226690/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2796819/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2796819/fulltext.xml /content/software_related_papers/PMC2796819/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC2376761/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4168749/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4168749/fulltext.xml /content/software_related_papers/PMC4168749/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5266326/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5266326/fulltext.xml /content/software_related_papers/PMC5266326/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC7194485/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC7194485/fulltext.xml /content/software_related_papers/PMC7194485/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2427162/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2427162/fulltext.xml /content/software_related_papers/PMC2427162/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4426829/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4426829/fulltext.xml /content/software_related_papers/PMC4426829/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2958747/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2958747/fulltext.xml /content/software_related_papers/PMC2958747/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6628208/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6628208/fulltext.xml /content/software_related_papers/PMC6628208/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5638226/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5638226/fulltext.xml /content/software_related_papers/PMC5638226/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3888149/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4706054/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4706054/fulltext.xml /content/software_related_papers/PMC4706054/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3307106/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3307106/fulltext.xml /content/software_related_papers/PMC3307106/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3967109/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3967109/fulltext.xml /content/software_related_papers/PMC3967109/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6880260/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6880260/fulltext.xml /content/software_related_papers/PMC6880260/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3962008/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3962008/fulltext.xml /content/software_related_papers/PMC3962008/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC6392199/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6042825/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6042825/fulltext.xml /content/software_related_papers/PMC6042825/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4264639/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4264639/fulltext.xml /content/software_related_papers/PMC4264639/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5167065/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5167065/fulltext.xml /content/software_related_papers/PMC5167065/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4411368/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4411368/fulltext.xml /content/software_related_papers/PMC4411368/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3228863/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3228863/fulltext.xml /content/software_related_papers/PMC3228863/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6492420/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6492420/fulltext.xml /content/software_related_papers/PMC6492420/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC6345265/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3685583/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5472867/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5472867/fulltext.xml /content/software_related_papers/PMC5472867/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4176421/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4176421/fulltext.xml /content/software_related_papers/PMC4176421/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3582273/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3582273/fulltext.xml /content/software_related_papers/PMC3582273/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4236746/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4236746/fulltext.xml /content/software_related_papers/PMC4236746/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4224201/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4224201/fulltext.xml /content/software_related_papers/PMC4224201/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3009535/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3009535/fulltext.xml /content/software_related_papers/PMC3009535/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3599859/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3599859/fulltext.xml /content/software_related_papers/PMC3599859/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4670012/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4670012/fulltext.xml /content/software_related_papers/PMC4670012/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6821346/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6821346/fulltext.xml /content/software_related_papers/PMC6821346/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3651407/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3651407/fulltext.xml /content/software_related_papers/PMC3651407/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3307116/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3307116/fulltext.xml /content/software_related_papers/PMC3307116/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6717989/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6717989/fulltext.xml /content/software_related_papers/PMC6717989/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3654709/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3654709/fulltext.xml /content/software_related_papers/PMC3654709/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4380025/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4380025/fulltext.xml /content/software_related_papers/PMC4380025/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3065683/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC2638934/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6013758/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6013758/fulltext.xml /content/software_related_papers/PMC6013758/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2530886/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2530886/fulltext.xml /content/software_related_papers/PMC2530886/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3646686/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3646686/fulltext.xml /content/software_related_papers/PMC3646686/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3530913/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3530913/fulltext.xml /content/software_related_papers/PMC3530913/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2752617/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2752617/fulltext.xml /content/software_related_papers/PMC2752617/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5260057/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5260057/fulltext.xml /content/software_related_papers/PMC5260057/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3413384/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4642848/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4642848/fulltext.xml /content/software_related_papers/PMC4642848/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2971582/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2971582/fulltext.xml /content/software_related_papers/PMC2971582/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5127930/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5127930/fulltext.xml /content/software_related_papers/PMC5127930/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5726607/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5726607/fulltext.xml /content/software_related_papers/PMC5726607/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4141640/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4141640/fulltext.xml /content/software_related_papers/PMC4141640/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4826505/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4826505/fulltext.xml /content/software_related_papers/PMC4826505/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4197741/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4197741/fulltext.xml /content/software_related_papers/PMC4197741/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2910028/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2910028/fulltext.xml /content/software_related_papers/PMC2910028/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3649846/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC5082607/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC5082607/fulltext.xml /content/software_related_papers/PMC5082607/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC2666812/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC2666812/fulltext.xml /content/software_related_papers/PMC2666812/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6051191/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6051191/fulltext.xml /content/software_related_papers/PMC6051191/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC3659198/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC6306265/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC6306265/fulltext.xml /content/software_related_papers/PMC6306265/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC4760270/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC4760270/fulltext.xml /content/software_related_papers/PMC4760270/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33m/content/software_related_papers/PMC5978649/fulltext.xml is empty\u001b[0m\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3521214/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3521214/fulltext.xml /content/software_related_papers/PMC3521214/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3750528/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3750528/fulltext.xml /content/software_related_papers/PMC3750528/sections\n",
            "\u001b[1;30mWARNING:\u001b[0m \u001b[33mMaking sections in /content/software_related_papers/PMC3835703/fulltext.xml\u001b[0m\n",
            "\u001b[1;30mINFO:\u001b[0m wrote XML sections for /content/software_related_papers/PMC3835703/fulltext.xml /content/software_related_papers/PMC3835703/sections\n",
            "  3% 358/13456 [00:00<00:12, 1082.47it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC3855388/sections/0_front/1_article-meta/21_funding-group/0_funding-statement/0_ext-link.xml\u001b[0m\n",
            "  9% 1169/13456 [00:01<00:11, 1090.47it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5624584/sections/0_front/1_article-meta/24_custom-meta-group/0_custom-meta/1_meta-value/1_ext-link.xml\u001b[0m\n",
            "\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5624584/sections/0_front/1_article-meta/24_custom-meta-group/0_custom-meta/1_meta-value/0_ext-link.xml\u001b[0m\n",
            " 23% 3114/13456 [00:02<00:09, 1067.93it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC4277354/sections/0_front/1_article-meta/22_custom-meta-group/0_custom-meta/1_meta-value/1_ext-link.xml\u001b[0m\n",
            "\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC4277354/sections/0_front/1_article-meta/22_custom-meta-group/0_custom-meta/1_meta-value/0_ext-link.xml\u001b[0m\n",
            " 51% 6891/13456 [00:05<00:05, 1246.23it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5266326/sections/0_front/1_article-meta/22_funding-group/4_funding-statement/0_ext-link.xml\u001b[0m\n",
            " 56% 7589/13456 [00:06<00:04, 1236.87it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5638226/sections/0_front/1_article-meta/19_funding-group/3_funding-statement/2_ext-link.xml\u001b[0m\n",
            "\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5638226/sections/0_front/1_article-meta/19_funding-group/3_funding-statement/1_ext-link.xml\u001b[0m\n",
            "\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC5638226/sections/0_front/1_article-meta/19_funding-group/3_funding-statement/0_ext-link.xml\u001b[0m\n",
            " 96% 12930/13456 [00:10<00:00, 1328.67it/s]\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC6306265/sections/0_front/1_article-meta/21_custom-meta-group/0_custom-meta/1_meta-value/1_ext-link.xml\u001b[0m\n",
            "\u001b[1;30mERROR:\u001b[0m \u001b[31mcannot parse /content/software_related_papers/PMC6306265/sections/0_front/1_article-meta/21_custom-meta-group/0_custom-meta/1_meta-value/0_ext-link.xml\u001b[0m\n",
            "100% 13456/13456 [00:10<00:00, 1229.25it/s]\n",
            "\u001b[1;30mINFO:\u001b[0m Found 31632 sentences in the section(s).\n",
            "\u001b[1;30mINFO:\u001b[0m getting terms from /content/software_related_papers/software_dictionary\n",
            "/usr/local/lib/python3.7/dist-packages/docanalysis/entity_extraction.py:316: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
            "  \"[\", \"\").str.replace(\"]\", \"\")\n",
            "\u001b[1;30mINFO:\u001b[0m wrote output to /content/software_related_papers/software.csv\n"
          ]
        }
      ]
    }
  ]
}