measurements/million_scale_measurements.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Probing part 1\n",
                "\n",
                "Vantage points will probe either the targets themselves (step 2) or other addresses in the same /24 prefix (step 2).\n",
                "\n",
                "Vantage points are only the anchors.  \n",
                "As always, targets are the anchors.  \n",
                "\n",
                "This notebook is an implementation of the million scale method. Check the paper for more information.  \n",
                "To do after create_datasets.ipynb"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import uuid\n",
                "\n",
                "from logger import logger\n",
                "from scripts.utils.file_utils import load_json\n",
                "from scripts.utils.measurement_utils import (\n",
                "    load_targets,\n",
                "    load_vps,\n",
                "    get_measurement_config,\n",
                "    save_measurement_config,\n",
                "    get_target_prefixes,\n",
                "    ping_prefixes,\n",
                "    ping_targets,\n",
                ")\n",
                "from default import (\n",
                "    USER_ANCHORS_FILE,\n",
                "    USER_HITLIST_FILE,\n",
                "    MEASUREMENT_CONFIG_PATH,\n",
                ")\n",
                "\n",
                "# will define the number of vps and targets to use\n",
                "NB_TARGETS = 2\n",
                "NB_VPS = 4"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Load targets and vps dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {},
            "outputs": [],
            "source": [
                "targets = load_targets(USER_ANCHORS_FILE, nb_target=NB_TARGETS)\n",
                "vps = load_vps(USER_ANCHORS_FILE, nb_vps=NB_VPS)\n",
                "\n",
                "# every anchors /24 subnet\n",
                "target_addrs = [t[\"address_v4\"] for t in targets]\n",
                "target_prefixes = get_target_prefixes(target_addrs)\n",
                "\n",
                "# responsive IP addresses in all /24 prefixes\n",
                "targets_per_prefix = load_json(USER_HITLIST_FILE)\n",
                "\n",
                "logger.info(f\"nb targets: {len(targets)}\")\n",
                "logger.info(f\"nb_vps : {len(vps)}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Generate measurement config\n",
                "\n",
                "This configuration is used to retrieve all measurements results from RIPE Atlas using their API."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-04 13:13:06::INFO:root:3171606573:: Starting experiment with uuid : c78efe35-8089-41a9-9206-ac7bac4a8a68\n"
                    ]
                }
            ],
            "source": [
                "# measurement configuration for retrieval\n",
                "experiment_uuid = str(uuid.uuid4())\n",
                "target_measurement_uuid = str(uuid.uuid4())\n",
                "prefix_measurement_uuid = str(uuid.uuid4())\n",
                "\n",
                "config_file_path = MEASUREMENT_CONFIG_PATH / f\"{experiment_uuid}.json\"\n",
                "\n",
                "logger.info(f\"Starting experiment with uuid : {experiment_uuid}\")\n",
                "\n",
                "measurement_config = get_measurement_config(\n",
                "    experiment_uuid=experiment_uuid,\n",
                "    target_measurement_uuid=target_measurement_uuid,\n",
                "    prefix_measurement_uuid=prefix_measurement_uuid,\n",
                "    targets=targets,\n",
                "    target_prefixes=target_prefixes,\n",
                "    vps=vps,\n",
                ")\n",
                "\n",
                "save_measurement_config(measurement_config, config_file_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Step 1: probing each target prefixes"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Probe target prefixes\n",
                "WARNING : Time consumming section"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-04 13:13:06::INFO:root:measurement_utils:: No cached results available\n",
                        "2024-10-04 13:13:06::INFO:root:measurement_utils:: Starting measurements dd2e9428-762d-4353-99ca-613057d430a3 with parameters: dry_run=False; nb_targets=2; nb_vps=4.\n",
                        "2024-10-04 13:13:06::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942232\n",
                        "2024-10-04 13:13:08::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942233\n",
                        "2024-10-04 13:13:08::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942234\n",
                        "2024-10-04 13:13:08::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942235\n",
                        "2024-10-04 13:13:09::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942236\n",
                        "2024-10-04 13:13:09::INFO:root:ping_and_traceroute_classes:: measurement tag: dd2e9428-762d-4353-99ca-613057d430a3 : started measurement id : 79942237\n",
                        "2024-10-04 13:13:09::INFO:root:ping_and_traceroute_classes:: measurement : dd2e9428-762d-4353-99ca-613057d430a3 done\n"
                    ]
                }
            ],
            "source": [
                "ping_prefixes(\n",
                "    measurement_uuid=prefix_measurement_uuid,\n",
                "    measurement_config=measurement_config,\n",
                "    target_prefixes=target_prefixes,\n",
                "    targets_per_prefix=targets_per_prefix,\n",
                "    vps=vps,\n",
                ")\n",
                "\n",
                "save_measurement_config(measurement_config, config_file_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Step 2: probing each target"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Probe targets\n",
                "WARNING : Time consumming section"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-04 13:13:11::INFO:root:measurement_utils:: Starting measurements 6796bfe3-7137-43f1-9f9f-71e0a141157d with parameters: dry_run=False; nb_targets=6; nb_vps=4.\n",
                        "2024-10-04 13:13:12::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942244\n",
                        "2024-10-04 13:13:12::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942245\n",
                        "2024-10-04 13:13:12::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942246\n",
                        "2024-10-04 13:13:13::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942247\n",
                        "2024-10-04 13:13:13::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942248\n",
                        "2024-10-04 13:13:14::INFO:root:ping_and_traceroute_classes:: measurement tag: 6796bfe3-7137-43f1-9f9f-71e0a141157d : started measurement id : 79942249\n",
                        "2024-10-04 13:13:14::INFO:root:ping_and_traceroute_classes:: measurement : 6796bfe3-7137-43f1-9f9f-71e0a141157d done\n"
                    ]
                }
            ],
            "source": [
                "# measurement configuration for retrieval\n",
                "ping_targets(\n",
                "    measurement_uuid=target_measurement_uuid,\n",
                "    measurement_config=measurement_config,\n",
                "    targets=targets,\n",
                "    vps=vps,\n",
                "    use_cache=False,\n",
                ")\n",
                "save_measurement_config(measurement_config, config_file_path)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Retrieve prefix results\n",
                "WARNING : Time consuming section\n",
                "\n",
                "Note: it might take some time before measurement results are available through RIPE API. If no results are available, retry after a few minutes (or hours, it might really depends on the probe itself)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [],
            "source": [
                "from logger import logger\n",
                "from scripts.utils.file_utils import load_json\n",
                "from scripts.utils.measurement_utils import (\n",
                "    retrieve_results,\n",
                "    insert_prefix_results,\n",
                "    insert_target_results,\n",
                ")\n",
                "from default import (\n",
                "    PREFIX_MEASUREMENT_RESULTS,\n",
                "    TARGET_MEASUREMENT_RESULTS,\n",
                ")\n",
                "\n",
                "# will define the number of vps and targets to use\n",
                "NB_TARGETS = 2\n",
                "NB_VPS = 4"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-04 13:13:14::INFO:root:3539837011:: {'experiment_uuid': 'c78efe35-8089-41a9-9206-ac7bac4a8a68', 'status': 'ongoing', 'start_time': '2024-10-04 13:13:06.112516', 'end_time': None, 'is_dry_run': False, 'nb_targets': 2, 'nb_vps': 4, 'description': 'measurements from a set of vps towards all targets/target prefixes', 'af': 4, 'target_measurements': {'measurement_uuid': '6796bfe3-7137-43f1-9f9f-71e0a141157d', 'targets': ['103.196.37.98', '195.246.236.1', '77.220.233.1', '185.230.79.16', '185.34.2.114', '217.25.179.62'], 'vps': {'77.220.233.1': {'id': 6824, 'address_v4': '77.220.233.1', 'asn_v4': 42699, 'country_code': 'DE', 'geometry': {'type': 'Point', 'coordinates': [13.7285, 51.0395]}}, '185.230.79.16': {'id': 7122, 'address_v4': '185.230.79.16', 'asn_v4': 204515, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.1585, 48.7085]}}, '185.34.2.114': {'id': 6798, 'address_v4': '185.34.2.114', 'asn_v4': 36236, 'country_code': 'AE', 'geometry': {'type': 'Point', 'coordinates': [55.8115, 25.6315]}}, '217.25.179.62': {'id': 7042, 'address_v4': '217.25.179.62', 'asn_v4': 24776, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.3695, 48.9085]}}}, 'end_time': 1728047594.2916105, 'start_time': 1728047591.8001034}, 'prefix_measurements': {'measurement_uuid': 'dd2e9428-762d-4353-99ca-613057d430a3', 'targets': ['103.196.37.0', '195.246.236.0'], 'vps': {'77.220.233.1': {'id': 6824, 'address_v4': '77.220.233.1', 'asn_v4': 42699, 'country_code': 'DE', 'geometry': {'type': 'Point', 'coordinates': [13.7285, 51.0395]}}, '185.230.79.16': {'id': 7122, 'address_v4': '185.230.79.16', 'asn_v4': 204515, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.1585, 48.7085]}}, '185.34.2.114': {'id': 6798, 'address_v4': '185.34.2.114', 'asn_v4': 36236, 'country_code': 'AE', 'geometry': {'type': 'Point', 'coordinates': [55.8115, 25.6315]}}, '217.25.179.62': {'id': 7042, 'address_v4': '217.25.179.62', 'asn_v4': 24776, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.3695, 48.9085]}}}, 'end_time': 1728047589.574289, 'start_time': 1728047586.1349247}, 'meshed_measurements': {'measurement_uuid': '805d6778-9e09-4be7-9c43-d4aafc813a10', 'targets': ['103.196.37.98', '195.246.236.1', '77.220.233.1', '185.230.79.16', '185.34.2.114', '217.25.179.62'], 'vps': {'77.220.233.1': {'id': 6824, 'address_v4': '77.220.233.1', 'asn_v4': 42699, 'country_code': 'DE', 'geometry': {'type': 'Point', 'coordinates': [13.7285, 51.0395]}}, '185.230.79.16': {'id': 7122, 'address_v4': '185.230.79.16', 'asn_v4': 204515, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.1585, 48.7085]}}, '185.34.2.114': {'id': 6798, 'address_v4': '185.34.2.114', 'asn_v4': 36236, 'country_code': 'AE', 'geometry': {'type': 'Point', 'coordinates': [55.8115, 25.6315]}}, '217.25.179.62': {'id': 7042, 'address_v4': '217.25.179.62', 'asn_v4': 24776, 'country_code': 'FR', 'geometry': {'type': 'Point', 'coordinates': [2.3695, 48.9085]}}}, 'end_time': 1728047591.7847333, 'start_time': 1728047589.5959833}}\n"
                    ]
                }
            ],
            "source": [
                "measurement_config = load_json(config_file_path)\n",
                "logger.info(measurement_config)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-04 13:13:14::INFO:root:1680719454:: retrieving results for measurement ids: dd2e9428-762d-4353-99ca-613057d430a3\n",
                        "2024-10-04 13:13:15::INFO:root:measurement_utils:: nb measurements retrieved: 0 for measurement_uuid : dd2e9428-762d-4353-99ca-613057d430a3\n"
                    ]
                },
                {
                    "ename": "UnboundLocalError",
                    "evalue": "local variable 'result' referenced before assignment",
                    "output_type": "error",
                    "traceback": [
                        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
                        "\u001b[0;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
                        "Cell \u001b[0;32mIn[9], line 8\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# sometimes, not all probes give output, reduce timeout if you do not want to wait for too long\u001b[39;00m\n\u001b[1;32m      6\u001b[0m response \u001b[38;5;241m=\u001b[39m retrieve_results(prefix_measurement_uuid, PREFIX_MEASUREMENT_RESULTS)\n\u001b[0;32m----> 8\u001b[0m \u001b[43minsert_prefix_results\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n",
                        "File \u001b[0;32m/storage/hugo/geoloc-imc-2023/scripts/utils/measurement_utils.py:324\u001b[0m, in \u001b[0;36minsert_prefix_results\u001b[0;34m(results)\u001b[0m\n\u001b[1;32m    319\u001b[0m values_description \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    320\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msrc, dst, prb_id, date, sent, rcvd, rtts, min, mean, msm_id, proto\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    321\u001b[0m )\n\u001b[1;32m    323\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m results:\n\u001b[0;32m--> 324\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mno data to insert, data = \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mresult\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    326\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m result \u001b[38;5;129;01min\u001b[39;00m results:\n\u001b[1;32m    327\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    328\u001b[0m         \u001b[38;5;66;03m# parse response\u001b[39;00m\n",
                        "\u001b[0;31mUnboundLocalError\u001b[0m: local variable 'result' referenced before assignment"
                    ]
                }
            ],
            "source": [
                "prefix_measurement_uuid = measurement_config[\"prefix_measurements\"][\"measurement_uuid\"]\n",
                "\n",
                "logger.info(f\"retrieving results for measurement ids: {prefix_measurement_uuid}\")\n",
                "\n",
                "# sometimes, not all probes give output, reduce timeout if you do not want to wait for too long\n",
                "response = retrieve_results(prefix_measurement_uuid, PREFIX_MEASUREMENT_RESULTS)\n",
                "\n",
                "insert_prefix_results(response)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Retrieve traget results\n",
                "WARNING : Time consumming section\n",
                "\n",
                "Note: it might take some time before measurement results are available through RIPE API. If no results are available, retry after a few minutes (or hours, it might really depends on the probe itself)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "2024-10-03 18:08:53::INFO:root:3802694766:: retrieving results for measurement ids: 18020ef4-fcc5-410b-9eb1-9ab3a18dd3a3\n",
                        "2024-10-03 18:08:53::INFO:root:measurement_utils:: nb measurements retrieved: 20 for measurement_uuid : 18020ef4-fcc5-410b-9eb1-9ab3a18dd3a3\n",
                        "2024-10-03 18:08:53::INFO:root:measurement_utils:: Target measurements successfully inserted in table : user_vps_to_target\n"
                    ]
                }
            ],
            "source": [
                "target_measurement_uuid = measurement_config[\"target_measurements\"][\"measurement_uuid\"]\n",
                "\n",
                "logger.info(f\"retrieving results for measurement ids: {target_measurement_uuid}\")\n",
                "\n",
                "response = retrieve_results(target_measurement_uuid, TARGET_MEASUREMENT_RESULTS)\n",
                "\n",
                "insert_target_results(response)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "geoloc-imc-2023-GZT64Hva-py3.10",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.12"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}