dist_diff_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "243518 releases\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "dist_diff = json.load(open(\"data/sampled_dist_diff.json\"))\n",
    "print(len(dist_diff), \"releases\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216741 (0.890040982596769) releases whose bdist files are all in sdist\n"
     ]
    }
   ],
   "source": [
    "emplty_bdist_only_pkgs = []\n",
    "\n",
    "for name, diffs in dist_diff.items():\n",
    "    sdist_only = diffs[\"sdist-only\"]\n",
    "    bdist_only = diffs[\"bdist-only\"]\n",
    "    if len(bdist_only) == 0:\n",
    "        emplty_bdist_only_pkgs.append(name)\n",
    "\n",
    "print(\n",
    "    f\"{len(emplty_bdist_only_pkgs)} ({len(emplty_bdist_only_pkgs) / len(dist_diff)}) releases whose bdist files are all in sdist\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3751\n"
     ]
    }
   ],
   "source": [
    "suffix_dist_names = {}\n",
    "for name, diffs in dist_diff.items():\n",
    "    bdist_only = diffs[\"bdist-only\"]\n",
    "    for fnames in bdist_only.values():\n",
    "        for f in fnames:\n",
    "            suffix = f.split(\"/\")[-1].split(\".\")[-1]\n",
    "            suffix_dist_names[suffix] = suffix_dist_names.get(suffix, [])\n",
    "            suffix_dist_names[suffix].append(name)\n",
    "\n",
    "suffix_dist_cnt = {k: len(set(v)) for k, v in suffix_dist_names.items()}\n",
    "suffix_dist_cnt = sorted(suffix_dist_cnt.items(), key=lambda x: x[1], reverse=True)\n",
    "print(len(suffix_dist_cnt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('py', 13310),\n",
       " ('pyc', 11007),\n",
       " ('so', 3748),\n",
       " ('pyd', 2191),\n",
       " ('txt', 790),\n",
       " ('json', 711),\n",
       " ('js', 648),\n",
       " ('html', 567),\n",
       " ('css', 409),\n",
       " ('png', 406)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suffix_dist_cnt[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107628 42499\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('__init__.py', 14246),\n",
       " ('utils.py', 680),\n",
       " ('models.py', 594),\n",
       " ('main.py', 411),\n",
       " ('base.py', 410),\n",
       " ('__main__.py', 378),\n",
       " ('version.py', 351),\n",
       " ('_version.py', 343),\n",
       " ('config.py', 303),\n",
       " ('core.py', 273)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "bdist_pyfiles = []\n",
    "bdist_pyfile_pkg_cnt = {}\n",
    "for name, diffs in dist_diff.items():\n",
    "    bdist_only = diffs[\"bdist-only\"]\n",
    "    py_files = []\n",
    "    for fnames in bdist_only.values():\n",
    "        for f in fnames:\n",
    "            if f.endswith(\".py\"):\n",
    "                fn = f.split(\"/\")[-1]\n",
    "                bdist_pyfiles.append(fn)\n",
    "                bdist_pyfile_pkg_cnt[fn] = bdist_pyfile_pkg_cnt.get(fn, [])\n",
    "                bdist_pyfile_pkg_cnt[fn].append(name)\n",
    "\n",
    "print(len(bdist_pyfiles), len(bdist_pyfile_pkg_cnt))\n",
    "Counter(bdist_pyfiles).most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('__init__.py', 3727),\n",
       " ('utils.py', 497),\n",
       " ('main.py', 335),\n",
       " ('__main__.py', 308),\n",
       " ('_version.py', 294),\n",
       " ('version.py', 244),\n",
       " ('test.py', 236),\n",
       " ('base.py', 221),\n",
       " ('cli.py', 215),\n",
       " ('config.py', 214)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bdist_pyfile_pkg_cnt = {k: len(set(v)) for k, v in bdist_pyfile_pkg_cnt.items()}\n",
    "bdist_pyfile_pkg_cnt = sorted(\n",
    "    bdist_pyfile_pkg_cnt.items(), key=lambda x: x[1], reverse=True\n",
    ")\n",
    "bdist_pyfile_pkg_cnt[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "radar",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}