src/analysis/github.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GitHub analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import seaborn as sns\n",
    "import tol_colors as tc\n",
    "from datetime import datetime\n",
    "from datetime import timezone\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Engagement numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stars_df = pd.read_csv(\"../data/analysis/stars.csv\", index_col=0)\n",
    "stars_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = np.hstack([0, 10**(np.arange(0,6))])\n",
    "counts, bins, _ = plt.hist(stars_df.groupby(\"github_user_cleaned_url\").count()[\"user\"], bins=bins, ec=\"black\", alpha=0.7)\n",
    "plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(counts)\n",
    "print(bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks_df = pd.read_csv(\"../data/analysis/forks.csv\", index_col=0)\n",
    "forks_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = np.hstack([0, 10**(np.arange(0,5))])\n",
    "counts, bins, _ = plt.hist(forks_df.groupby(\"github_user_cleaned_url\").count()[\"user\"], bins = bins, ec=\"black\", alpha=0.7)\n",
    "plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fork_counts = forks_df.groupby(\"github_user_cleaned_url\")[\"user\"].count()\n",
    "fork_counts.rename(\"forks_no\", inplace=True)\n",
    "star_counts = stars_df.groupby(\"github_user_cleaned_url\")[\"user\"].count()\n",
    "star_counts.rename(\"stars_no\", inplace=True)\n",
    "engagement = pd.merge(left=fork_counts, right=star_counts, on=\"github_user_cleaned_url\")\n",
    "engagement.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(engagement[\"forks_no\"], engagement[\"stars_no\"], alpha=0.5)\n",
    "plt.xlabel(\"Number of forks\")\n",
    "plt.ylabel(\"Number of stars\")\n",
    "plt.xscale('log')\n",
    "plt.yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cell_text = [\n",
    "    [f\"{fork_counts.mean():.2f}\", f\"{fork_counts.std():.2f}\", f\"{fork_counts.median():.2f}\", f\"{fork_counts.min():.2f}\", f\"{fork_counts.max():.2f}\"],\n",
    "    [f\"{star_counts.mean():.2f}\", f\"{star_counts.std():.2f}\", f\"{star_counts.median():.2f}\", f\"{star_counts.min():.2f}\", f\"{star_counts.max():.2f}\"]\n",
    "    ]\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "# hide axes\n",
    "#fig.patch.set_visible(False)\n",
    "ax.axis('off')\n",
    "ax.axis('tight')\n",
    "\n",
    "table = plt.table(cellText=cell_text,\n",
    "                  rowLabels=[\"forks\", \"stars\"],\n",
    "                  colLabels=[\"mean\", \"std\", \"median\", \"min\", \"max\"])\n",
    "\n",
    "fig.tight_layout()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Timelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_against_days_since_creation(df, column_name, label):\n",
    "    \"\"\"Plots number of markers set on all days since creation (total, one year, one month).\n",
    "\n",
    "    Args:\n",
    "        df (pd.DataFrame): input dataframe\n",
    "        column_name (str): name of column with days since creation info\n",
    "        label (str): what number we are looking at\n",
    "    \"\"\"\n",
    "    counted = df.groupby(column_name).count()\n",
    "    fig = plt.figure(figsize=(20, 4))\n",
    "    plt.subplot(131)\n",
    "    plt.bar(counted.index, counted.github_user_cleaned_url)\n",
    "    plt.xlabel(\"days since repository creation\")\n",
    "    plt.ylabel(f\"number of {label} on that day\")\n",
    "    plt.title(\"Complete timeline across all repositories\")\n",
    "\n",
    "    plt.subplot(132)\n",
    "    plt.bar(counted.index[:365], counted.github_user_cleaned_url[:365])\n",
    "    plt.xlabel(\"days since repository creation\")\n",
    "    plt.ylabel(f\"number of {label} on that day\")\n",
    "    plt.title(\"First year\")\n",
    "\n",
    "    plt.subplot(133)\n",
    "    plt.bar(counted.index[:31], counted.github_user_cleaned_url[:31])\n",
    "    plt.xlabel(\"days since repository creation\")\n",
    "    plt.ylabel(f\"number of {label} on that day\")\n",
    "    plt.title(\"First month\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_against_weeks_since_creation(df, column_name, label):\n",
    "    \"\"\"Plots number of markers set on all days since creation (total, one year, one month).\n",
    "\n",
    "    Args:\n",
    "        df (pd.DataFrame): input dataframe\n",
    "        column_name (str): name of column with days since creation info\n",
    "        label (str): what number we are looking at\n",
    "    \"\"\"\n",
    "    df[f\"weekly_{column_name}\"] = df[column_name]//7\n",
    "    counted = df.groupby(f\"weekly_{column_name}\").count()\n",
    "    fig = plt.figure(figsize=(20, 4))\n",
    "    plt.subplot(131)\n",
    "    plt.bar(counted.index, counted.github_user_cleaned_url)\n",
    "    plt.xlabel(\"weeks since repository creation\")\n",
    "    plt.ylabel(f\"number of label on that day\")\n",
    "    plt.title(\"Complete timeline across all repositories\")\n",
    "\n",
    "    plt.subplot(132)\n",
    "    plt.bar(counted.index[:52], counted.github_user_cleaned_url[:52])\n",
    "    plt.xlabel(\"weeks since repository creation\")\n",
    "    plt.ylabel(f\"number of label on that day\")\n",
    "    plt.title(\"First year\")\n",
    "\n",
    "    plt.subplot(133)\n",
    "    plt.bar(counted.index[:4], counted.github_user_cleaned_url[:4])\n",
    "    plt.xlabel(\"weeks since repository creation\")\n",
    "    plt.ylabel(f\"number of label on that day\")\n",
    "    plt.title(\"First month\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"../data/analysis\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_csv(os.path.join(data_dir, \"metadata.csv\"), index_col=0)\n",
    "metadata.created_at = pd.to_datetime(metadata.created_at, utc=True)\n",
    "metadata.rename(columns={\"created_at\": \"repo_created_at\"}, inplace=True)\n",
    "stars = pd.read_csv(os.path.join(data_dir, \"stars.csv\"), index_col=0)\n",
    "stars.date = pd.to_datetime(stars.date, utc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(metadata, stars, on=\"github_user_cleaned_url\")\n",
    "df.dropna(subset=[\"date\"], inplace=True)\n",
    "df[\"starred_on_day_since_creation\"] = (df.date - df.repo_created_at).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_against_days_since_creation(df, \"starred_on_day_since_creation\", \"stars\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_against_weeks_since_creation(df, \"starred_on_day_since_creation\", \"stars\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks = pd.read_csv(os.path.join(data_dir, \"forks.csv\"), index_col=0)\n",
    "forks.date = pd.to_datetime(forks.date, utc=True)\n",
    "df = pd.merge(metadata, forks, on=\"github_user_cleaned_url\")\n",
    "df.dropna(subset=[\"date\"], inplace=True)\n",
    "df[\"forked_on_day_since_creation\"] = (df.date - df.repo_created_at).dt.days\n",
    "plot_against_days_since_creation(df, \"forked_on_day_since_creation\", \"forks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_against_weeks_since_creation(df, \"forked_on_day_since_creation\", \"forks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issues = pd.read_csv(os.path.join(data_dir, \"issues.csv\"), index_col=0)\n",
    "issues.created_at = pd.to_datetime(issues.created_at, utc=True)\n",
    "issues.closed_at = pd.to_datetime(issues.closed_at, utc=True)\n",
    "df = pd.merge(metadata, issues, on=\"github_user_cleaned_url\")\n",
    "df.dropna(subset=[\"created_at\"], inplace=True)\n",
    "df[\"issue_opened_on_day_since_creation\"] = (df.created_at - df.repo_created_at).dt.days\n",
    "df = df[df.issue_opened_on_day_since_creation >= 0]\n",
    "plot_against_days_since_creation(df, \"issue_opened_on_day_since_creation\", \"issues opened\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_against_weeks_since_creation(df, \"issue_opened_on_day_since_creation\", \"issues opened\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## file presence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"../data/analysis/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents = pd.read_csv(os.path.join(data_dir, \"contents.csv\"), index_col=0)\n",
    "contents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents[pd.notna(contents.citation_added) == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.notna(contents.contributing_added).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.loc[pd.notna(contents.contributing_added)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents[contents.readme_emojis > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents_df = pd.merge(metadata, contents, on=\"github_user_cleaned_url\")\n",
    "contents_df[\"citation_added\"] = pd.to_datetime(contents_df.citation_added, utc=True)\n",
    "contents_df[\"citation_added\"] = (contents_df.citation_added - contents_df.created_at).dt.days // 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in contents.license.unique():\n",
    "    tmp = contents[contents.license == l]\n",
    "    plt.scatter(contents.readme_size, contents.contributing_size, alpha=0.3, marker='.', label=l)\n",
    "plt.xlabel(\"Size of README file\")\n",
    "plt.ylabel(\"Size of CONTRIBUTING.md\")\n",
    "plt.xlim(-1000, 30000)\n",
    "plt.ylim(-10, 300)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.license = contents.license.fillna('None')\n",
    "contents.license.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "permissive_licenses = [\"mit\", \"gpl-3.0\", \"apache-2.0\", \"bsd-3-clause\", \"gpl-2.0\", \"bsd-2-clause\"] # https://en.wikipedia.org/wiki/Permissive_software_license"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.license = contents.license.fillna('None')\n",
    "contents[\"license_type\"] = np.where(contents.license.isin(permissive_licenses), \"permissive\", np.where(contents.license == \"None\", \"None\", np.where(contents.license == \"other\", \"unknown\", \"non-permissive\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.license_type.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.github_user_cleaned_url[contents.license == \"other\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.github_user_cleaned_url[contents.license == \"bsd-3-clause\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.license = contents.license.fillna('None')\n",
    "contents.plot(x=\"license\", y=\"readme_size\", kind=\"scatter\", alpha=0.3, rot=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### README size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_size_series = contents.set_index(\"github_user_cleaned_url\").readme_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 300, 1500, 10000]\n",
    "binmeanings = [\"none\", \"ultra-short\", \"short\", \"informative\", \"detailed\"]\n",
    "if readme_size_series.max() > bins[-1]:\n",
    "    bins.append(readme_size_series.max())\n",
    "counts, bins = np.histogram(readme_size_series, bins)\n",
    "binlabels = [f\"{binmeanings[i]}\\n[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "binlabels += [f\"{binmeanings[-1]}\\n[{bins[-2]} - {bins[-1]}]\"]\n",
    "fig, ax = plt.subplots(ncols=1, nrows=1)#, figsize=(18, 12))\n",
    "ax.bar(binlabels, counts)\n",
    "ax.bar_label(ax.containers[0])\n",
    "ax.tick_params(axis='x', labelrotation=45)\n",
    "ax.set(xlabel=\"size of README in Bytes\", ylabel=\"repository count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 300, 1500, 10000]\n",
    "binmeanings = [\"none\", \"ultra-short\", \"short\", \"informative\", \"detailed\"]\n",
    "if readme_size_series.max() > bins[-1]:\n",
    "    bins.append(readme_size_series.max())\n",
    "counts, bins = np.histogram(readme_size_series, bins)\n",
    "binlabels = [f\"{binmeanings[i]}\\n[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "binlabels += [f\"{binmeanings[-1]}\\n[{bins[-2]} - {bins[-1]}]\"]\n",
    "fig, ax = plt.subplots(ncols=1, nrows=1)#, figsize=(18, 12))\n",
    "ax.pie(counts, labels=binlabels)\n",
    "#ax.bar(binlabels, counts)\n",
    "#ax.bar_label(ax.containers[0])\n",
    "#ax.tick_params(axis='x', labelrotation=45)\n",
    "#ax.set(xlabel=\"size of README in Bytes\", ylabel=\"repository count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 300, 1500, 10000]\n",
    "if readme_size_series.max() > bins[-1]:\n",
    "    bins.append(readme_size_series.max())\n",
    "lower = bins[0]\n",
    "for upper in bins[1:]:\n",
    "    tmp = readme_size_series[readme_size_series.between(lower, upper)]\n",
    "    tmp = tmp.sort_values()\n",
    "    samples_low = tmp.iloc[:3]\n",
    "    samples_high = tmp.iloc[-3:]\n",
    "    print((lower, upper))\n",
    "    print(\"lower:\")\n",
    "    print(samples_low)\n",
    "    print(\"higher:\")\n",
    "    print(samples_high)\n",
    "    print()\n",
    "    lower = upper\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series = readme_size_series[readme_size_series.between(6000, 15000)]\n",
    "plt.scatter(x=series, y=[1]*len(series), alpha=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### connect with engagement numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks = pd.read_csv(os.path.join(data_dir, \"forks.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks_count = forks.groupby(\"github_user_cleaned_url\").date.count().rename(\"no_forks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(contents, forks_count, left_on=\"github_user_cleaned_url\", right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.plot(\n",
    "    kind=\"scatter\",\n",
    "    x=\"license_type\",\n",
    "    y=\"no_forks\",\n",
    "    alpha=0.5\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Team size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_csv(os.path.join(data_dir, \"metadata.csv\"), index_col=0)\n",
    "metadata[\"created_at\"] = pd.to_datetime(metadata.created_at)\n",
    "contributions = pd.read_csv(os.path.join(data_dir, \"contributions.csv\"), index_col=0)\n",
    "contributions[\"week_co\"] = pd.to_datetime(contributions.week_co)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contrib_df = pd.merge(metadata[[\"github_user_cleaned_url\", \"created_at\"]], contributions)\n",
    "contrib_df[\"week_since_repo_creation\"] = (contrib_df.week_co - contrib_df.created_at).dt.days // 7\n",
    "team_df = contrib_df[[\"github_user_cleaned_url\", \"author\", \"week_since_repo_creation\", \"commits\"]].set_index([\"github_user_cleaned_url\", \"author\", \"week_since_repo_creation\"]).sort_index()\n",
    "# user is active contributor if made at least one commit in last 12 weeks\n",
    "windowed_team_df = team_df.groupby(level=\"author\").rolling(window=12, min_periods=0).sum().droplevel(0)\n",
    "windowed_team_df[\"active contributors\"] = windowed_team_df.commits > 0\n",
    "# team size\n",
    "team_size = windowed_team_df.groupby(level=[\"github_user_cleaned_url\", \"week_since_repo_creation\"])[\"active contributors\"].value_counts()[:,:,True]\n",
    "max_team_size = team_size.groupby(level=\"github_user_cleaned_url\").max()\n",
    "max_team_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = max_team_size.hist()\n",
    "ax.bar_label(ax.containers[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### team size vs license"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents = pd.read_csv(os.path.join(data_dir, \"contents.csv\"), index_col=0)\n",
    "permissive_licenses = [\"mit\", \"gpl-3.0\", \"apache-2.0\", \"bsd-3-clause\", \"gpl-2.0\", \"bsd-2-clause\"] # https://en.wikipedia.org/wiki/Permissive_software_license\n",
    "contents.license = contents.license.fillna('None')\n",
    "contents[\"license_type\"] = np.where(\n",
    "    contents.license.isin(permissive_licenses), \"permissive\", np.where(\n",
    "    contents.license == \"None\", \"None\", np.where(\n",
    "    contents.license == \"other\", \"unknown\", \"non-permissive\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(max_team_size, contents[[\"github_user_cleaned_url\", \"license_type\", \"license\"]], left_index=True, right_on=\"github_user_cleaned_url\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(df.license_type.unique())\n",
    "fig, axs = plt.subplots(ncols=n, nrows=1, figsize=(6*n, 6))\n",
    "for i, lt in enumerate(df.license_type.unique()):\n",
    "    ax = axs[i]\n",
    "    bins = [1, 2, 5, 10]\n",
    "    tmp = df[df[\"license_type\"] == lt]\n",
    "    if tmp[\"active contributors\"].max() > bins[-1]:\n",
    "        bins.append(tmp[\"active contributors\"].max())\n",
    "    counts, bins = np.histogram(tmp[\"active contributors\"], bins)\n",
    "    binlabels = [f\"[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "    binlabels += [f\"[{bins[-2]} - {bins[-1]}]\"]\n",
    "    ax.bar(binlabels, counts)\n",
    "    ax.bar_label(ax.containers[0])\n",
    "    ax.set(xlabel=\"maximum team size\", ylabel=\"repository count\", title=lt)\n",
    "plt.suptitle(\"Team size per license type\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(df.license_type.unique())\n",
    "fig, axs = plt.subplots(ncols=n, nrows=1, figsize=(6*n, 6))\n",
    "for i, lt in enumerate(df.license_type.unique()):\n",
    "    ax = axs[i]\n",
    "    bins = [1, 2, 5, 10]\n",
    "    tmp = df[df[\"license_type\"] == lt]\n",
    "    if tmp[\"active contributors\"].max() > bins[-1]:\n",
    "        bins.append(tmp[\"active contributors\"].max())\n",
    "    counts, bins = np.histogram(tmp[\"active contributors\"], bins)\n",
    "    binlabels = [f\"[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "    binlabels += [f\"[{bins[-2]} - {bins[-1]}]\"]\n",
    "    ax.pie(x=counts, labels=binlabels, autopct='%1.1f%%')\n",
    "    #ax.bar_label(ax.containers[0])\n",
    "    #ax.set(xlabel=\"maximum team size\", ylabel=\"repository count\", title=lt)\n",
    "    ax.set(title=f\"{lt} ({len(tmp)} repos)\")\n",
    "plt.suptitle(\"Maximum team size per license type\")\n",
    "plt.savefig(\"../data/analysis/overall/team_size_per_license_type.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 2, 5, 10]\n",
    "if df[\"active contributors\"].max() > bins[-1]:\n",
    "    bins.append(df[\"active contributors\"].max())\n",
    "counts, bins = np.histogram(df[\"active contributors\"], bins)\n",
    "binlabels = [f\"[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "binlabels += [f\"[{bins[-2]} - {bins[-1]}]\"]\n",
    "\n",
    "n = len(bins)-2\n",
    "fig, axs = plt.subplots(ncols=n, nrows=1, figsize=(6*n, 6))\n",
    "lower=bins[1]\n",
    "# iterate\n",
    "for i, upper in enumerate(bins[2:-1]):\n",
    "    ax = axs[i]\n",
    "    tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] < upper)]\n",
    "    tmp.license_type.value_counts().sort_index().plot(\n",
    "        kind='bar',\n",
    "        ax=ax,\n",
    "        xlabel=\"license type\",\n",
    "        ylabel=\"repository count\",\n",
    "        title=\"team size \"+binlabels[1+i]\n",
    "    )\n",
    "    lower = upper\n",
    "# add last with inclusive upper limit\n",
    "ax = axs[-1]\n",
    "upper = bins[-1]\n",
    "tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] <= upper)]\n",
    "tmp.license_type.value_counts().sort_index().plot(\n",
    "    kind='bar',\n",
    "    ax=ax,\n",
    "    xlabel=\"license type\",\n",
    "    ylabel=\"repository count\",\n",
    "    title=\"team size \"+binlabels[-1]\n",
    ")\n",
    "plt.suptitle(\"License type per team size\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 2, 5, 10]\n",
    "if df[\"active contributors\"].max() > bins[-1]:\n",
    "    bins.append(df[\"active contributors\"].max())\n",
    "counts, bins = np.histogram(df[\"active contributors\"], bins)\n",
    "binlabels = [f\"[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "binlabels += [f\"[{bins[-2]} - {bins[-1]}]\"]\n",
    "\n",
    "n = len(bins)-2\n",
    "fig, axs = plt.subplots(ncols=n, nrows=1, figsize=(6*n, 6))\n",
    "lower=bins[1]\n",
    "# iterate\n",
    "for i, upper in enumerate(bins[2:-1]):\n",
    "    ax = axs[i]\n",
    "    tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] < upper)]\n",
    "    tmp.license.value_counts().sort_index().plot(\n",
    "        kind='bar',\n",
    "        ax=ax,\n",
    "        xlabel=\"license\",\n",
    "        ylabel=\"repository count\",\n",
    "        title=\"team size \"+binlabels[1+i]\n",
    "    )\n",
    "    lower = upper\n",
    "# add last with inclusive upper limit\n",
    "ax = axs[-1]\n",
    "upper = bins[-1]\n",
    "tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] <= upper)]\n",
    "tmp.license.value_counts().sort_index().plot(\n",
    "    kind='bar',\n",
    "    ax=ax,\n",
    "    xlabel=\"license\",\n",
    "    ylabel=\"repository count\",\n",
    "    title=\"team size \"+binlabels[-1]\n",
    ")\n",
    "plt.suptitle(\"License type per team size\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 1, 2, 5, 10]\n",
    "if df[\"active contributors\"].max() > bins[-1]:\n",
    "    bins.append(df[\"active contributors\"].max())\n",
    "counts, bins = np.histogram(df[\"active contributors\"], bins)\n",
    "binlabels = [f\"[{bins[i]} - {bins[i+1]})\" for i in range(len(bins)-2)]\n",
    "binlabels += [f\"[{bins[-2]} - {bins[-1]}]\"]\n",
    "\n",
    "n = len(bins)-2\n",
    "fig, axs = plt.subplots(ncols=n, nrows=1, figsize=(6*n, 6))\n",
    "lower=bins[1]\n",
    "# iterate\n",
    "for i, upper in enumerate(bins[2:-1]):\n",
    "    ax = axs[i]\n",
    "    tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] < upper)]\n",
    "    tmp.license_type.value_counts().sort_index().plot(\n",
    "        kind='pie',\n",
    "        ax=ax,\n",
    "        #xlabel=\"license type\",\n",
    "        ylabel=\"\",\n",
    "        title=f\"team size {binlabels[1+i]} ({len(tmp)} repos)\",\n",
    "        autopct='%1.1f%%'\n",
    "    )\n",
    "    lower = upper\n",
    "# add last with inclusive upper limit\n",
    "ax = axs[-1]\n",
    "upper = bins[-1]\n",
    "tmp = df[(df[\"active contributors\"] >= lower) & (df[\"active contributors\"] <= upper)]\n",
    "tmp.license_type.value_counts().sort_index().plot(\n",
    "    kind='pie',\n",
    "    ax=ax,\n",
    "    #xlabel=\"license type\",\n",
    "    ylabel=\"\",\n",
    "    title=f\"team size {binlabels[-1]} ({len(tmp)} repos)\",\n",
    "    autopct='%1.1f%%'\n",
    ")\n",
    "plt.suptitle(\"License type per team size\")\n",
    "plt.savefig(\"../data/analysis/overall/license_type_per_team_size.png\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## README headlines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_df = pd.read_csv(os.path.join(data_dir, \"readme_history.csv\"), index_col=0)\n",
    "readme_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_df[readme_df.added_headings.str.contains(\"Neo4J 2.0.1\", na=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import ast\n",
    "\n",
    "pattern = \"\\[(.+?)\\]\\(.+?\\)\"\n",
    "text = 'COVID-19 image data collection ([🎬 video about the project](https://www.youtube.com/watch?v=ineWmqfelEQ))'\n",
    "re.sub(pattern, r'\\1', text, count=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import re\n",
    "\n",
    "def clean_heading(h):\n",
    "    to_remove = string.digits + string.whitespace + \".:\"\n",
    "    h = h.lstrip(to_remove)\n",
    "    pattern = \"\\[(.+?)\\]\\(.+?\\)\"\n",
    "    h = re.sub(pattern, r'\\1', h, count=0)\n",
    "    h = h.replace(string.punctuation, \"\")\n",
    "    h = h.strip(string.punctuation)\n",
    "    h = h.lower()\n",
    "    return h\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "\n",
    "headings = []\n",
    "for l in readme_df.added_headings.dropna():\n",
    "    headings += ast.literal_eval(l)\n",
    "headings = [clean_heading(h) for h in headings]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_df.added_cites[(readme_df.added_cites != \"[]\") & (readme_df.added_cites.notna())]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### word cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "stopwords = STOPWORDS\n",
    "custom = set([\"trades\", \"glosat\", \"glosat_table_dataset\", \"nilmtk\", \"bert\", \"lemon\", \"cascadetabnet\"])\n",
    "stopwords = stopwords.union(custom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordcloud = WordCloud(\n",
    "    collocation_threshold=15,\n",
    "    stopwords=stopwords,\n",
    "    scale=10,\n",
    "    background_color=\"white\",\n",
    "    random_state=42\n",
    "    ).generate(\" \".join(headings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20, 10))\n",
    "plt.imshow(wordcloud)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### timelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_headings(df):\n",
    "    interesting_words = {\n",
    "        \"ownership\": [\"license\", \"example\", \"reference\", \"citation\", \"cited\", \"publication\", \"paper\"],\n",
    "        \"usage\": [\"requirements\", \"using\", \"example\", \"usage\", \"run\", \"install\", \"installing\", \"installation\", \"tutorial\", \"tutorials\", \"build\", \"guide\", \"documentation\"]\n",
    "    }\n",
    "    df[\"ownership_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"ownership\"]))\n",
    "    df[\"usage_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"usage\"]))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_df.author_date = pd.to_datetime(readme_df.author_date, utc=True)\n",
    "df = pd.merge(metadata, readme_df, on=\"github_user_cleaned_url\")\n",
    "df.dropna(subset=[\"author_date\"], inplace=True)\n",
    "df[\"authored_on_day_since_creation\"] = (df.author_date - df.repo_created_at).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = analyse_headings(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df[\"ownership_addition\"] | df[\"usage_addition\"]].head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## example: rOpenHealth/ClinicalCodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(data_dir, filename, repo, to_datetime=None):\n",
    "    df = pd.read_csv(os.path.join(data_dir, filename), index_col=0)\n",
    "    df = df[df[\"github_user_cleaned_url\"] == repo]\n",
    "    if type(to_datetime) == list:\n",
    "        for dt in to_datetime:\n",
    "            df[dt] = pd.to_datetime(df[dt], utc=True)\n",
    "    elif type(to_datetime) == str:\n",
    "        df[to_datetime] = pd.to_datetime(df[to_datetime], utc=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"../data/analysis\"\n",
    "repo = \"esbmc/esbmc\"\n",
    "contents = load_data(data_dir, \"contents.csv\", repo, [\"citation_added\", \"contributing_added\"])\n",
    "contributions = load_data(data_dir, \"contributions.csv\", repo, \"week_co\")\n",
    "forks = load_data(data_dir, \"forks.csv\", repo, \"date\")\n",
    "issues = load_data(data_dir, \"issues.csv\", repo, [\"created_at\", \"closed_at\"])\n",
    "metadata = load_data(data_dir, \"metadata.csv\", repo, \"created_at\")\n",
    "readme_history = load_data(data_dir, \"readme_history.csv\", repo, \"author_date\")\n",
    "stars = load_data(data_dir, \"stars.csv\", repo, \"date\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User type wrt. issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end = (datetime.now(tz=timezone.utc) - metadata.created_at.iloc[0]).days // 7\n",
    "x_data = pd.Series(np.arange(end), name=\"week_since_repo_creation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = pd.merge(issues, metadata, on=\"github_user_cleaned_url\", suffixes=(None,\"_repo\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df[\"created_at\"] = (merged_df[\"created_at\"] - merged_df[\"created_at_repo\"]).dt.days // 7\n",
    "merged_df[\"closed_at\"] = (merged_df[\"closed_at\"] - merged_df[\"created_at_repo\"]).dt.days // 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "created = merged_df.groupby([\"user\", \"created_at\"])[\"state\"].count().rename(\"created_count\")\n",
    "created.index.rename({\"created_at\": \"week_since_repo_creation\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "closed = merged_df.groupby([\"closed_by\", \"closed_at\"])[\"state\"].count().rename(\"closed_count\")\n",
    "closed.index.rename({\"closed_at\": \"week_since_repo_creation\", \"closed_by\": \"user\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issues_by_user = pd.merge(created, closed, left_index=True, right_index=True, how=\"outer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(x_data, pd.Series(issues_by_user.index.unique(level=\"user\")), how=\"cross\").set_index([\"user\", \"week_since_repo_creation\"])\n",
    "df = pd.merge(df, issues_by_user, left_index=True, right_index=True, how=\"outer\")\n",
    "df.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "windowed_df = df.groupby(level=\"user\").rolling(window=12, min_periods=0).sum().droplevel(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = pd.IndexSlice\n",
    "windowed_df.loc[idx[:, 0], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conditions = [(windowed_df.created_count > 0) & (windowed_df.closed_count == 0), (windowed_df.created_count == 0) & (windowed_df.closed_count > 0), (windowed_df.created_count > 0) & (windowed_df.closed_count > 0)]\n",
    "choices = [\"opening\", \"closing\", \"both\"]\n",
    "windowed_df[\"status\"] = np.select(conditions, choices, default=\"inactive\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cset_light = tc.tol_cset('light')\n",
    "cset_light"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.cm.register_cmap('rainbow_discrete_12', tc.tol_cmap('rainbow_discrete', 12))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "palette = sns.color_palette(\"rainbow_discrete_12\", n_colors=12)\n",
    "palette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "palette_short = [palette[i] for i in range(len(palette)) if i in [1, 4, 6, 9]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(20, 4))\n",
    "sns.scatterplot(\n",
    "    ax=ax,\n",
    "    data=windowed_df,\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=\"user\",\n",
    "    hue=\"status\",\n",
    "    hue_order=[\"inactive\", \"opening\", \"closing\", \"both\"],\n",
    "    palette=palette_short,\n",
    "    marker=\"|\",\n",
    "    s=500,\n",
    "    )\n",
    "left, right = ax.get_xlim()\n",
    "ax.set_xlim(left=0, right=right+10)\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Team size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contrib_df = pd.merge(metadata[[\"github_user_cleaned_url\", \"created_at\"]], contributions)\n",
    "contrib_df[\"week_since_repo_creation\"] = (contrib_df.week_co - contrib_df.created_at).dt.days // 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_df = contrib_df[[\"author\", \"week_since_repo_creation\", \"commits\"]].set_index([\"author\", \"week_since_repo_creation\"]).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "windowed_team_df = team_df.groupby(level=\"author\").rolling(window=12, min_periods=0).sum().droplevel(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "windowed_team_df[\"active contributor\"] = windowed_team_df.commits > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(20, 4))\n",
    "sns.scatterplot(\n",
    "    ax=ax,\n",
    "    data=windowed_team_df,\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=\"author\",\n",
    "    hue=\"active contributor\",\n",
    "    hue_order=[False, True],\n",
    "    palette=['#d62728', '#2ca02c'],\n",
    "    marker=\"|\",\n",
    "    s=500,\n",
    "    )\n",
    "left, right = ax.get_xlim()\n",
    "ax.set_ylabel(\"user\")\n",
    "ax.set_xlim(left=0, right=right+10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_size = windowed_team_df.groupby(level=\"week_since_repo_creation\")[\"active contributor\"].value_counts()[:,True].reindex(windowed_team_df.index.levels[1], fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_size.plot(\n",
    "    figsize=(20, 4),\n",
    "    xlabel=\"week since repo creation\",\n",
    "    ylabel=\"contributor team size\",\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "became_active = team_df.groupby(level=\"author\").cumsum()\n",
    "became_active[\"contributor\"] = became_active.commits > 0\n",
    "became_active"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_growth = became_active.groupby(level=\"week_since_repo_creation\")[\"contributor\"].value_counts()[:,True].reindex(became_active.index.levels[1], fill_value=0)\n",
    "#windowed_team_df[\"active contributor\"] = windowed_team_df.commits > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_growth.plot(\n",
    "    figsize=(20, 4),\n",
    "    xlabel=\"week since repo creation\",\n",
    "    ylabel=\"contributor team size\",\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Issues opened and closed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issues_timeline_df = pd.merge(metadata, issues, on=\"github_user_cleaned_url\", suffixes=(\"_repo\", None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issues_timeline_df[\"opened_in_week_since_repo_creation\"] = (issues_timeline_df.created_at - issues_timeline_df.created_at_repo).dt.days // 7\n",
    "issues_timeline_df[\"closed_in_week_since_repo_creation\"] = (issues_timeline_df.closed_at - issues_timeline_df.created_at_repo).dt.days // 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issues_timeline_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end = (datetime.now(tz=timezone.utc) - metadata.created_at.iloc[0]).days // 7\n",
    "x_data = pd.Series(np.arange(end), name=\"week_since_repo_creation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_count_timeline = pd.DataFrame(x_data)\n",
    "issue_count_timeline[\"open_issues_count\"] = issue_count_timeline.apply(lambda x: len(issues_timeline_df[\n",
    "                                                                                        (issues_timeline_df.opened_in_week_since_repo_creation <= x.week_since_repo_creation) &\n",
    "                                                                                        ((issues_timeline_df.closed_in_week_since_repo_creation >= x.week_since_repo_creation) |\n",
    "                                                                                         (issues_timeline_df.closed_in_week_since_repo_creation.isna()))\n",
    "                                                                                        ]), axis=1)\n",
    "issue_count_timeline[\"closed_issues_count\"] = issue_count_timeline.apply(lambda x: len(issues_timeline_df[\n",
    "                                                                                        (issues_timeline_df.closed_in_week_since_repo_creation < x.week_since_repo_creation)\n",
    "                                                                                        ]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_count_timeline.rename(columns={\"open_issues_count\": \"open issues\", \"closed_issues_count\": \"closed issues\"}).plot(\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=[\"open issues\", \"closed issues\"],\n",
    "    xlabel=\"week since repo creation\",\n",
    "    ylabel=\"count\"\n",
    "    )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_issues_count_p = np.diff(issue_count_timeline['open_issues_count']) / np.diff(issue_count_timeline['week_since_repo_creation'])\n",
    "week_since_repo_creation_p = (np.array(issue_count_timeline['week_since_repo_creation'])[:-1] + np.array(issue_count_timeline['week_since_repo_creation'])[1:]) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plt.plot(issue_count_timeline['week_since_repo_creation'], issue_count_timeline['open_issues_count'], label=\"open issues\")\n",
    "plt.plot(week_since_repo_creation_p, open_issues_count_p, label=\"diff open issues\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Engagement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks_df = pd.merge(forks, metadata, on=\"github_user_cleaned_url\")\n",
    "forks_df[\"week_since_repo_creation\"] = (forks_df.date - forks_df.created_at).dt.days // 7\n",
    "forks_df = forks_df[[\"week_since_repo_creation\", \"user\"]].groupby(\"week_since_repo_creation\").count().rename(columns={\"user\": \"no_forks\"}).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stars_df = pd.merge(stars, metadata, on=\"github_user_cleaned_url\")\n",
    "stars_df[\"week_since_repo_creation\"] = (stars_df.date - stars_df.created_at).dt.days // 7\n",
    "stars_df = stars_df[[\"week_since_repo_creation\", \"user\"]].groupby(\"week_since_repo_creation\").count().rename(columns={\"user\": \"no_stars\"}).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end = (datetime.now(tz=timezone.utc) - metadata.created_at.iloc[0]).days // 7\n",
    "x_data = pd.Series(np.arange(end), name=\"week_since_repo_creation\")\n",
    "engagement_df = pd.merge(x_data, forks_df, on=\"week_since_repo_creation\", how=\"outer\")\n",
    "engagement_df = pd.merge(engagement_df, stars_df, on=\"week_since_repo_creation\", how=\"outer\").fillna(0)\n",
    "engagement_df = engagement_df.set_index(\"week_since_repo_creation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "engagement_df = engagement_df.cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "engagement_df.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diffs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_diff_series(x, y):\n",
    "    y_p = np.diff(y) / np.diff(x)\n",
    "    x_p = (np.array(x)[:-1] + np.array(x)[1:]) / 2\n",
    "    return (x_p, y_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_p, y_p = get_diff_series(team_growth.index, team_growth)\n",
    "plt.plot(x_p, y_p, label=\"diff team growth\")\n",
    "x_p, y_p = get_diff_series(engagement_df.index, engagement_df[\"no_stars\"])\n",
    "plt.plot(x_p, y_p, label=\"diff stars\")\n",
    "x_p, y_p = get_diff_series(engagement_df.index, engagement_df[\"no_forks\"])\n",
    "plt.plot(x_p, y_p, label=\"diff forks\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Useless if not fitting curve first."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Highlights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import re\n",
    "\n",
    "def clean_heading(h):\n",
    "    to_remove = string.digits + string.whitespace + \".:\"\n",
    "    h = h.lstrip(to_remove)\n",
    "    pattern = \"\\[(.+?)\\]\\(.+?\\)\"  # markdown links\n",
    "    h = re.sub(pattern, r'\\1', h, count=0)\n",
    "    h = h.replace(string.punctuation, \"\")\n",
    "    h = h.strip(string.punctuation)\n",
    "    h = h.lower()\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "\n",
    "headings = readme_history.added_headings.dropna().apply(ast.literal_eval).explode().dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headings = headings.apply(clean_heading)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_headings(df):\n",
    "    interesting_words = {\n",
    "        \"ownership\": [\"license\", \"example\", \"reference\", \"citation\", \"cited\", \"publication\", \"paper\"],\n",
    "        \"usage\": [\"requirements\", \"using\", \"example\", \"usage\", \"run\", \"install\", \"installing\", \"installation\", \"tutorial\", \"tutorials\", \"build\", \"guide\", \"documentation\"]\n",
    "    }\n",
    "    df[\"ownership_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"ownership\"]), case=False)\n",
    "    df[\"usage_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"usage\"]), case=False)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(metadata, readme_history, on=\"github_user_cleaned_url\")\n",
    "df.dropna(subset=[\"author_date\"], inplace=True)\n",
    "df[\"authored_in_week_since_creation\"] = ((df.author_date - df.created_at).dt.days // 7)#.clip(0)\n",
    "#df[\"authored_in_week_since_creation\"].clip(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = analyse_headings(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_count_timeline.rename(columns={\"open_issues_count\": \"open issues\", \"closed_issues_count\": \"closed issues\"}).plot(\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=[\"open issues\", \"closed issues\"],\n",
    "    xlabel=\"week since repo creation\",\n",
    "    ylabel=\"count\"\n",
    "    )\n",
    "ownership_added = df[df.ownership_addition].authored_in_week_since_creation\n",
    "plt.scatter(ownership_added, (-2 * np.ones((len(ownership_added),))), marker=\"v\", color=\"black\", label=\"ownership heading\")\n",
    "usage_added = df[df.usage_addition].authored_in_week_since_creation\n",
    "plt.scatter(usage_added, (-1 * np.ones((len(usage_added),))), marker=\"v\", color=\"red\", label=\"usage heading\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_headings(df):\n",
    "    interesting_words = {\n",
    "        \"ownership\": [\"license\", \"example\", \"reference\", \"citation\", \"cited\", \"publication\", \"paper\"],\n",
    "        \"usage\": [\"requirements\", \"using\", \"example\", \"usage\", \"run\", \"install\", \"installing\", \"installation\", \"tutorial\", \"tutorials\", \"build\", \"guide\", \"documentation\"]\n",
    "    }\n",
    "    df[\"ownership_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"ownership\"]), case=False)\n",
    "    df[\"usage_addition\"] = df.added_headings.str.contains(\"|\".join(interesting_words[\"usage\"]), case=False)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def date_highlights(readme_history, contents, metadata, ax):\n",
    "    df = pd.merge(metadata, readme_history, on=\"github_user_cleaned_url\")\n",
    "    df.dropna(subset=[\"author_date\"], inplace=True)\n",
    "    df[\"authored_in_week_since_creation\"] = (df.author_date - df.created_at).dt.days // 7\n",
    "    contents_df = pd.merge(metadata, contents, on=\"github_user_cleaned_url\")\n",
    "    contents_df.citation_added = (contents_df.citation_added - contents_df.created_at).dt.days // 7\n",
    "    contents_df.contributing_added = (contents_df.contributing_added - contents_df.created_at).dt.days // 7\n",
    "    # headings\n",
    "    df = analyse_headings(df)\n",
    "    ownership_added = df[df.ownership_addition].authored_in_week_since_creation\n",
    "    ax.scatter(ownership_added, (1 * np.ones((len(ownership_added),))), marker=\"v\", s=100, label=\"ownership heading\")\n",
    "    usage_added = df[df.usage_addition].authored_in_week_since_creation\n",
    "    ax.scatter(usage_added, (2 * np.ones((len(usage_added),))), marker=\"v\", label=\"usage heading\")\n",
    "    # citation in README\n",
    "    citation_added = df[(df.added_cites != \"[]\") & (df.added_cites.notna())]\n",
    "    ax.scatter(citation_added, (3 * np.ones((len(citation_added),))), marker=\"v\", label=\"citation in README\")\n",
    "    # citation file\n",
    "    citation_file_added = contents_df[contents_df.citation_added.notna()]\n",
    "    ax.scatter(citation_file_added, (4* np.ones((len(citation_file_added),))), marker=\"v\", label=\"citation file\")\n",
    "    # contributing file\n",
    "    contributing_file_added = contents_df[contents_df.contributing_added.notna()]\n",
    "    ax.scatter(contributing_file_added, (5* np.ones((len(contributing_file_added),))), marker=\"v\", label=\"contributing file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1)\n",
    "date_highlights(readme_history, contents, metadata, ax)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_data = pd.read_csv(\"../data/analysis/cleaned_links/joined.csv\")\n",
    "pd.merge(metadata, paper_data, on=\"github_user_cleaned_url\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User fork/star highlight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = np.unique(np.concatenate([issues.user.unique(), issues.closed_by.dropna().unique(), contributions.author.unique()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks_users_df = forks[forks.user.isin(users)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forks_users_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stars_users_df = stars[stars.user.isin(users)]\n",
    "stars_users_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## test aggregated datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_id = \"ziqizhang/sti\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = {}\n",
    "dfs[\"overall_timeline_df\"] = pd.read_csv(\"../data/analysis/aggregated_timeline.csv\")\n",
    "dfs[\"commit_author_df\"] = pd.read_csv(\"../data/analysis/aggregated_commit_author_timeline.csv\")\n",
    "dfs[\"issue_user_df\"] = pd.read_csv(\"../data/analysis/aggregated_issue_user_timeline.csv\")\n",
    "dfs[\"overall_df\"] = pd.read_csv(\"../data/analysis/aggregated_overall.csv\")\n",
    "for k, v in dfs.items():\n",
    "    dfs[k] = v[v.github_user_cleaned_url == repo_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare figure\n",
    "fig = plt.figure(figsize=(20, 20))\n",
    "overlay_axis = fig.subplots()\n",
    "overlay_axis.axis('off')\n",
    "axs = fig.subplots(nrows=6, sharex=True, height_ratios=[3, 3, 2, 2, 2, 1])\n",
    "for ax in axs:\n",
    "    ax.patch.set_alpha(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_type_wrt_issues\n",
    "sns.scatterplot(\n",
    "    ax=axs[0],\n",
    "    data=dfs[\"issue_user_df\"],\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=\"user\",\n",
    "    hue=\"user_status\",\n",
    "    hue_order=[\"inactive\", \"opening\", \"closing\", \"both\"],\n",
    "    palette=['#d62728', '#1f77b4', '#ff7f0e', '#2ca02c'],\n",
    "    marker=\"|\",\n",
    "    s=500,\n",
    "    )\n",
    "axs[0].set_ylabel(\"issue user\")\n",
    "axs[0].legend(loc=\"upper right\")\n",
    "axs[0].grid(True, axis=\"x\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# contributor_team\n",
    "dfs[\"commit_author_df\"][\"active_contributors\"] = dfs[\"commit_author_df\"][\"active_contributors\"].map({True: \"active\", False: \"inactive\"})\n",
    "# plot per-user status\n",
    "sns.scatterplot(\n",
    "    ax=axs[1],\n",
    "    data=dfs[\"commit_author_df\"],\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=\"author\",\n",
    "    hue=\"active_contributors\",\n",
    "    hue_order=[\"inactive\", \"active\"],\n",
    "    palette=['#d62728', '#2ca02c'],\n",
    "    marker=\"|\",\n",
    "    s=500,\n",
    ")\n",
    "axs[1].set_ylabel(\"contributing user\")\n",
    "dfs[\"overall_timeline_df\"].plot(\n",
    "    ax=axs[2],\n",
    "    lw=2,\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=[\"active_contributors\", \"contributors\"],\n",
    "    ylabel=\"number of contributors\"\n",
    ")\n",
    "axs[1].grid(True, axis=\"x\")\n",
    "axs[1].legend()\n",
    "axs[2].legend(loc=\"upper right\")\n",
    "axs[2].grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# no_open_and_closed_issues\n",
    "dfs[\"overall_timeline_df\"].plot(\n",
    "    ax=axs[3],\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=[\"open_count\", \"closed_count\"],\n",
    "    ylabel=\"issues\"\n",
    ")\n",
    "axs[3].legend(loc=\"upper right\")\n",
    "axs[3].grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# engagement\n",
    "dfs[\"overall_timeline_df\"].plot(\n",
    "    ax=axs[4],\n",
    "    x=\"week_since_repo_creation\",\n",
    "    y=[\"forks_count\", \"stars_count\"],\n",
    "    ylabel=\"issues\"\n",
    ")\n",
    "axs[4].legend(loc=\"upper right\")\n",
    "axs[4].grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function for highlights\n",
    "def calc_y_timeline(data):\n",
    "    ys = [[] for _ in range(len(data))]\n",
    "    seen_x = []\n",
    "    for i in range(len(data)):\n",
    "        for x in data[i]:\n",
    "            ys[i].append(-1 * seen_x.count(x))\n",
    "            seen_x.append(x)\n",
    "    return ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# date_highlights\n",
    "ax=axs[5]\n",
    "ax.set(ylim=(-6, 0.4), yticks=[])\n",
    "ax.set_xlabel(\"weeks since repository creation\", loc=\"right\")\n",
    "ax.xaxis.set_label_position('top')\n",
    "ax.xaxis.tick_top()\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['bottom'].set_visible(False)\n",
    "ax.spines['left'].set_visible(False)\n",
    "events_df = dfs[\"overall_timeline_df\"]\n",
    "data = [events_df[events_df.ownership_added].week_since_repo_creation,\n",
    "        events_df[events_df.usage_added].week_since_repo_creation, \n",
    "        events_df[events_df.citation_added].week_since_repo_creation, \n",
    "        events_df[events_df.citation_file_added].week_since_repo_creation, \n",
    "        events_df[events_df.contributing_file_added].week_since_repo_creation, \n",
    "        events_df[events_df.paper_published].week_since_repo_creation]\n",
    "ys = calc_y_timeline(data)\n",
    "labels = [\"ownership heading\", \"usage heading\", \"citation in README\", \"citation file\", \"contributing file\", \"mention in publication\"]\n",
    "prop_cycle = plt.rcParams['axes.prop_cycle']\n",
    "colors = prop_cycle.by_key()['color']\n",
    "ymax = 86\n",
    "for i in range(len(data)):\n",
    "    ax.scatter(data[i], ys[i], marker=\"^\", s=100, label=labels[i], color=colors[i])\n",
    "    overlay_axis.vlines(data[i], ys[i], ymax, linestyles='dashed', color=colors[i])\n",
    "axs[5].legend(loc=\"upper right\", ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final adjustments\n",
    "ymax = 86\n",
    "xl, xr = plt.xlim()\n",
    "plt.xlim(xl, xr+15)\n",
    "overlay_axis.set(xlim=(xl, xr+15), ylim=(-6, ymax))\n",
    "fig.suptitle(repo_id)\n",
    "fig.tight_layout(rect=[0, 0.03, 1, 0.98])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ssi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}