multiprocess walk osmnx distances Wien bar-pub-cafe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5f78d5c0",
   "metadata": {},
   "source": [
    "# graph of walking distances between all pubs/bars/cafes in Wien\n",
    "\n",
    "create dataset of distances between all bar/pubs/cafes in Vienna which can be used to find optimized route for visiting one pub in each district (Bezirk) - inspired by https://www.seidlrallye.at/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7fcc5afe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "# get nodes with info from osm file - file generated on https://overpass-turbo.eu/\n",
    "def get_nodes(osm_filename):\n",
    "    tree = ET.parse(osm_filename)\n",
    "    root = tree.getroot()\n",
    "    nodes = []\n",
    "    for i in range(len(root)):\n",
    "        node = {}\n",
    "        #print(root[i].tag, root[i].attrib)\n",
    "        if root[i].tag != \"node\":\n",
    "            continue\n",
    "        node[\"id\"] = root[i].attrib[\"id\"]\n",
    "        node[\"lat\"] = root[i].attrib[\"lat\"]\n",
    "        node[\"lon\"] = root[i].attrib[\"lon\"]\n",
    "        for j in range(len(root[i])):\n",
    "            attrib = root[i][j].attrib\n",
    "            node[attrib[\"k\"]] = attrib[\"v\"]\n",
    "        nodes.append(node)\n",
    "    return nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "669e8155",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of nodes for bar, pub, cafe amenity from https://overpass-turbo.eu/\n",
    "nodes_bar = get_nodes('wien_bar.osm')\n",
    "nodes_pub = get_nodes('wien_pub.osm')\n",
    "nodes_cafe = get_nodes('wien_cafe.osm')\n",
    "all_nodes = nodes_bar + nodes_pub + nodes_cafe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c844638",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1041 entries, 0 to 1861\n",
      "Columns: 227 entries, id to snack\n",
      "dtypes: object(227)\n",
      "memory usage: 1.8+ MB\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame.from_records(all_nodes)\n",
    "df_postcode = df[df['addr:postcode'].notna()] # leave only those with PLZ so we can identify Bezirk\n",
    "df_postcode.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9c530273",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 987 entries, 20 to 1725\n",
      "Columns: 227 entries, id to snack\n",
      "dtypes: object(227)\n",
      "memory usage: 1.7+ MB\n"
     ]
    }
   ],
   "source": [
    "# leave only those inside of Wien\n",
    "\n",
    "df_w = pd.DataFrame()\n",
    "\n",
    "for i in range(1,24):\n",
    "    plz=\"1{:02d}0\".format(i)\n",
    "    df_w = pd.concat([df_w, df_postcode[df_postcode['addr:postcode']==plz]])\n",
    "    \n",
    "df_w.info()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6400bac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import isnan\n",
    "\n",
    "# make node dict, remove empty fields\n",
    "def get_clean_nodes_dict(df):\n",
    "    nodes_dict = {}\n",
    "    for row in df.to_dict(orient=\"records\"):\n",
    "        node = {k: row[k] for k in row if not (isinstance(row[k], float) and isnan(row[k]))}\n",
    "        nodes_dict[row[\"id\"]] = node\n",
    "    return nodes_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d8570451",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_dict = get_clean_nodes_dict(df_w)\n",
    "\n",
    "# node id as int\n",
    "nodes = [n for n in nodes_dict.values()]\n",
    "for n in nodes:\n",
    "    n['id'] = int(n['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b2cc49bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jan/venv39_2/lib/python3.9/site-packages/osmnx/utils.py:192: UserWarning: The `utils.config` function is deprecated and will be removed in a future release. Instead, use the `settings` module directly to configure a global setting's value. For example, `ox.settings.log_console=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# get map of Wien as a graph\n",
    "\n",
    "import osmnx as ox\n",
    "import matplotlib.pyplot as plt\n",
    "import networkx as nx\n",
    "from IPython.display import IFrame\n",
    "\n",
    "ox.config(log_console=True, use_cache=True)\n",
    "\n",
    "place_name = \"Vienna, Austria\"\n",
    "graph = ox.graph_from_place(place_name, network_type='walk')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "42dc0891",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9min 46s, sys: 126 ms, total: 9min 46s\n",
      "Wall time: 9min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# find nearest nodes in map based on coordinates\n",
    "osm_node_dict = {}\n",
    "\n",
    "for n in nodes:\n",
    "    osm_node_dict[n['id']] = ox.nearest_nodes(graph, float(n['lon']), float(n['lat']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "619b0636",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create list of from-to nodes for which we want to calculate distances\n",
    "\n",
    "osm_nodes = list(osm_node_dict.keys())\n",
    "#osm_nodes = osm_nodes[:50]\n",
    "from_to_distances_dict = {}\n",
    "\n",
    "for id1 in range(len(osm_nodes)):\n",
    "    for id2 in range(id1+1, len(osm_nodes)):\n",
    "        id1_node = osm_nodes[id1]\n",
    "        id2_node = osm_nodes[id2]\n",
    "        from_to_distances_dict[(id1_node, id2_node)]=-1\n",
    "        \n",
    "work = list(from_to_distances_dict.keys())        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ccddca6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_distance(nodes_ids):\n",
    "    (from_node, to_node) = nodes_ids\n",
    "    # shortest walking distance in meters\n",
    "    distance = nx.shortest_path_length(graph, osm_node_dict[from_node], osm_node_dict[to_node], weight='length')\n",
    "    return round(distance)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "c3f92359",
   "metadata": {},
   "source": [
    "%%time\n",
    "\n",
    "# get distances in parallel\n",
    "import multiprocessing\n",
    "from multiprocessing import Pool\n",
    "\n",
    "print(\"Number of cpu : \", multiprocessing.cpu_count())\n",
    "\n",
    "p = Pool(multiprocessing.cpu_count())\n",
    "distances = p.map(compute_distance, work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d1878689",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 486591/486591 [3:19:41<00:00, 40.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28min 3s, sys: 2min 42s, total: 30min 45s\n",
      "Wall time: 3h 19min 41s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# get distances in parallel with progressbar\n",
    "import multiprocessing\n",
    "from multiprocessing import Pool\n",
    "import tqdm\n",
    "\n",
    "pool = Pool(multiprocessing.cpu_count())\n",
    "distances = []\n",
    "for distance in tqdm.tqdm(pool.imap(func=compute_distance, iterable=work), total=len(work)):\n",
    "        distances.append(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4fb5c26b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine work and results\n",
    "for w, d in zip(work, distances):\n",
    "    from_to_distances_dict[w]=d\n",
    "    #print(w,d)\n",
    "\n",
    "# create nice dict which can be exported as json\n",
    "from_to_distances = []\n",
    "for k, v in from_to_distances_dict.items():\n",
    "    from_to_distance = {\"from\" : k[0], \"to\": k[1], 'distance' : v}\n",
    "    from_to_distances.append(from_to_distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "320b153e",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dict = {\n",
    "    \"nodes\" : nodes,\n",
    "    \"edges\" : from_to_distances\n",
    "}\n",
    "\n",
    "import json\n",
    "\n",
    "# save the json file\n",
    "with open(\"wien_bar-pub-cafe_walk_distances.json\", \"wt\") as f:\n",
    "    json.dump(final_dict, f, indent=4, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d9c8110a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'from': 566660740, 'to': 566660785, 'distance': 109},\n",
       " {'from': 566660740, 'to': 870980192, 'distance': 531},\n",
       " {'from': 566660740, 'to': 901058218, 'distance': 1184},\n",
       " {'from': 566660740, 'to': 956159131, 'distance': 946},\n",
       " {'from': 566660740, 'to': 956430135, 'distance': 644}]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dict['edges'][:5]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}