Merge pull request #149 from dioptra-io/main

version 1.1.2
dioptra-io · Dec 15, 2022 · 243dcc4 · 243dcc4
2 parents b9b082a + 3d750c3
commit 243dcc4
Show file tree

Hide file tree

Showing 16 changed files with 1,961 additions and 1,509 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.1
+current_version = 1.1.2
 commit = True
 tag = True
 message = release(project): {current_version} → {new_version}

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: "3.10"
       - uses: dioptra-io/setup-poetry-action@v1
       - name: Start services
         run: docker compose up -d -t 0 traefik clickhouse minio postgres redis
@@ -37,3 +37,19 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           # platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' && github.actor != 'dependabot[bot]' }}
+
+  mkdocs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: dioptra-io/setup-poetry-action@v1
+      - name: Install package
+        run: poetry install
+      - name: Build documentation
+        run: poetry run mkdocs build --strict
+      - name: Publish documentation
+        run: poetry run mkdocs gh-deploy --force --no-history --strict
+        if: ${{ github.ref == 'refs/heads/main' }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,33 +3,33 @@ repos:
     rev: v1.4
     hooks:
       - id: autoflake
-        args: [ "--in-place", "--remove-all-unused-imports" ]
+        args: ["--in-place", "--remove-all-unused-imports"]
 
   - repo: https://github.com/timothycrosley/isort
     rev: 5.10.1
     hooks:
       - id: isort
-        args: [ "--profile=black" ]
+        args: ["--profile=black"]
 
   - repo: https://github.com/psf/black
     rev: 22.3.0
     hooks:
       - id: black
-        args: [ "--line-length=88" ]
+        args: ["--line-length=88"]
 
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.2
     hooks:
       - id: flake8
         # https://github.com/PyCQA/pycodestyle/issues/373
         # E203: False positive "whitespace before ':' " on list slice.
-        args: [ "--ignore=E203,E501,W503" ]
+        args: ["--ignore=E203,E501,W503"]
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: ^README.md$
+        exclude: .md$
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files
diff --git a/README.md b/README.md
@@ -12,9 +12,9 @@ Think of it as a project similar to [CAIDA Ark](https://www.caida.org/projects/a
 
 We offer a public instance of Iris, as well as public measurement data, on [iris.dioptra.io](https://iris.dioptra.io).
 
-## 🚀 Deployment
+## 📖 Documentation
 
-See [`DEPLOYMENT.md`](DEPLOYMENT.md) for more information about how to deploy Iris on your own infrastructure.
+Please refer to the [documentation](https://dioptra-io.github.io/iris/) for more information on how to use Iris, deploy your own instance and contribute to the project.
 
 ## 📚 Publications
 
@@ -39,11 +39,6 @@ keywords = {active internet measurements, internet topology}
 }
 ```
 
-## ✏️ Contributing
-
-See [`CONTRIBUTING.md`](CONTRIBUTING.md) for more information about how to contribute to this project.
-
-
 ## 🧑‍💻 Authors
 
 Iris is developed and maintained by the [Dioptra group](https://dioptra.io) at [Sorbonne Université](https://www.sorbonne-universite.fr) in Paris, France.
diff --git a/configuration/clickhouse/init-db.sh b/configuration/clickhouse/init-db.sh
@@ -8,4 +8,7 @@ clickhouse client -n <<-EOSQL
     CREATE DATABASE IF NOT EXISTS iris_test;
     GRANT ALL ON iris.* TO iris WITH GRANT OPTION;
     GRANT ALL ON iris_test.* TO iris WITH GRANT OPTION;
+    -- These permissions are necessary for testing iris-exporter
+    -- without creating a dedicated iris-exporter user.
+    GRANT CREATE TEMPORARY TABLE, S3 ON *.* TO iris;
 EOSQL
diff --git a/docs/api.md b/docs/api.md
@@ -0,0 +1,92 @@
+# HTTP API
+
+Iris is meant to be controlled through an HTTP API.
+The API is documented with Swagger on the `/docs` endpoint.
+A documentation of the API is publicly available at [https://api.iris.dioptra.io/docs/](https://api.iris.dioptra.io/docs/).
+
+On this page we document specific tips for using the API.
+
+## Registering
+
+### On iris.dioptra.io
+
+To register on Iris public instance, you must go through [iris.dioptra.io](https://iris.dioptra.io).
+Upon registration, you will be invited to sign a license and send it to our team for approval.
+Once your account is approved you will be able to access public Iris data.
+
+### On a private instance
+
+To register on a private instance, you can use the `/auth/register` endpoint:
+```
+curl -X POST \
+    -H 'Content-Type: application/json' \
+    -d '{"firstname": "First", "lastname": "Last", "email": "[email protected]", "password": "abc"}' \
+    https://api.dev.iris.dioptra.io/auth/register
+```
+
+```json
+{
+  "id": "23ec99b5-f259-4732-b121-d012b686e37a",
+  "email": "[email protected]",
+  "is_active": true,
+  "is_superuser": false,
+  "is_verified": false,
+  "firstname": "First",
+  "lastname": "Last",
+  "probing_enabled": false,
+  "probing_limit": 1,
+  "allow_tag_reserved": false,
+  "allow_tag_public": false,
+  "creation_time": "2022-08-17T14:24:22.495581"
+}
+```
+
+An administrator can then set `is_verified` and `probing_enabled` to true by issuing a PATCH query against the `/users/:id` endpoint.
+
+## iris-client
+
+To make it easier to use the Iris API from Python code, you can use the [iris-client](https://github.com/dioptra-io/iris-client) library.
+It is implemented on top of [httpx](https://github.com/encode/httpx) and has sync and async interfaces.
+
+```bash
+pip install dioptra-iris-client
+```
+
+To avoid specifying the credentials in the code, you can use environment variables, or a configuration file:
+
+```json title="~/.config/iris/credentials.json"
+{
+    "base_url": "https://api.iris.dioptra.io",
+    "username": "[email protected]",
+    "password": "admin"
+}
+```
+
+```python
+from iris_client import IrisClient
+
+with IrisClient() as iris:
+    iris.all("/measurements/", params={"tag": "collection:public", "only_mine": False})
+    iris.post("/measurements/", json={"tool": "diamond-miner", "agents": "..."})
+```
+
+For more information, refer to the documentation of the library.
+
+## pych-client
+
+To access Iris data hosted on ClickHouse, you can use the [pych-client](https://github.com/dioptra-io/pych-client) library in combination with iris-client.
+
+```bash
+pip install pych-client
+```
+
+```python
+from iris_client import IrisClient
+from pych_client import ClickHouseClient
+
+with IrisClient() as iris:
+    services = iris.get("/users/me/services").json()
+    with ClickHouseClient(**services["clickhouse"]) as clickhouse:
+        print(clickhouse.json("SHOW TABLES"))
+
+```
diff --git a/docs/concepts.md b/docs/concepts.md
@@ -0,0 +1,59 @@
+# Concepts
+
+## Agent
+
+An agent performs measurements.
+It receives a list of probes to send and uses [caracal](https://github.com/dioptra-io/caracal) to send the probes and capture the replies.
+It is implemented as Python application that loops on a Redis queue and uploads the results to an S3 bucket.
+
+The resources required to run an agent are minimal and depends mostly on the desired probing rate.
+A minimum of 1 CPU and 512 MB of memory are required.
+To achieve a rate of 100k packets per second, we recommend at-least 2 CPU.
+
+If you're running an agent in the cloud, avoid burstable instances since the agent will exhaust the CPU credits very quickly and will become very slow.
+
+## Measurement
+
+A measurement is defined by a tool and a list of measurement agents.
+
+## Measurement agent
+
+A measurement agent is defined by an agent, a target list and tool parameters specific to the agent.
+
+## Target list
+
+A target list is a comma-delimited list of networks to probe.
+Each line of the file must be like
+```
+target,protocol,min_ttl,max_ttl,n_initial_flows
+```
+where the target is a IPv4/IPv6 prefix or IPv4/IPv6 address.
+The prococol can be `icmp`, `icmp6` or `udp`.
+The file name must end with `.csv`.
+
+For example:
+```
+0.0.0.0/0,udp,2,32,1
+8.8.8.0/24,icmp,2,32,6
+2001:4860:4860::8888,icmp6,2,32,6
+```
+If the prefix length is ignored, /24 or /128 is assumed.
+
+Some tools offer the `prefix_len_v4` and `prefix_len_v6` parameters which allows to split the specified networks and keep the target list short.
+For instance, if `prefix_len_v4=24`, then `0.0.0.0/0` will be split into the 16 millions networks `0.0.0.0/24,...,255.255.255.0/24`.
+
+## Tool
+
+A tool defines which probes should be sent based on a target list and the results of a previous measurement round.
+Examples of such tools are Diamond-Miner, Yarrp or ping.
+
+A better name would probably have been an _algorithm_, to avoid confusion with the actual probing tool that is used, [caracal](https://github.com/dioptra-io/caracal).
+
+## Worker
+
+A worker coordinates measurement agents.
+It runs the tool to get the list of probes to send, it sends this list to the agent, and it waits for the results.
+It is implemented as [Dramatiq](https://dramatiq.io) actors and uses Redis and S3 to exchange data with the agents.
+
+The resources required to run a worker depends on the tool and on the number of concurrent measurement agents.
+To run Diamond-Miner on `0.0.0.0/0` with a single agent, we recommend at-least 32 GB of memory and 8 CPUs.
diff --git a/DEPLOYMENT.md → docs/deployment.md b/DEPLOYMENT.md → docs/deployment.md
@@ -2,7 +2,7 @@
 
 You can set up a production-ready system to orchestrate multiple vantage points from a dedicated API.
 
-We provide a [docker-compose.yml](docker-compose.yml) file to set up Iris locally.
+We provide a [`docker-compose.yml`](https://github.com/dioptra-io/iris/blob/main/docker-compose.yml) file to set up Iris locally.
 Feel free to adapt it with your own configurations.
 Don't forget to change the default passwords before pushing it to production!
 
@@ -42,11 +42,11 @@ curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/js
 
 In this section we will document the different settings to configure Iris.
 Most of the settings are commons to the API, the worker and the agent components.
-All of the settings must be declared in the [docker-compose.yml](docker-compose.yml) file for each component.
+All of the settings must be declared in the [`docker-compose.yml`](https://github.com/dioptra-io/iris/blob/main/docker-compose.yml) file for each component.
 
 | Component    | Settings location                                |
 |--------------|--------------------------------------------------|
-| Commons  | [iris/commons/settings.py](iris/commons/settings.py) |
-| API      | [iris/api/settings.py](iris/api/settings.py)         |
-| Worker   | [iris/worker/settings.py](iris/worker/settings.py)   |
-| Agent    | [iris/agent/settings.py](iris/agent/settings.py)     |
+| Commons  | [iris/commons/settings.py](https://github.com/dioptra-io/iris/blob/main/iris/commons/settings.py) |
+| API      | [iris/api/settings.py](https://github.com/dioptra-io/iris/blob/main/iris/api/settings.py)         |
+| Worker   | [iris/worker/settings.py](https://github.com/dioptra-io/iris/blob/main/iris/worker/settings.py)   |
+| Agent    | [iris/agent/settings.py](https://github.com/dioptra-io/iris/blob/main/iris/agent/settings.py)     |
diff --git a/CONTRIBUTING.md → docs/dev.md b/CONTRIBUTING.md → docs/dev.md
@@ -1,10 +1,10 @@
-# Contributing
+# Development
 
-Thanks for contributing ! Here is some guidelines to make your life easier during the development process.
+Here are some guidelines to make your life easier during the development process.
 
 ## Prerequisites
 
-To develop on Iris you need a Python 3.8+ interpreter and Docker.
+To develop on Iris you need a Python 3.10+ interpreter and Docker.
 
 Iris services and their dependencies are hosted behind a Traefik reverse-proxy.
 To be able to access them from your own machine, you need to add the following entries to [`/etc/hosts`](file:///etc/hosts):
@@ -50,7 +50,8 @@ poetry run alembic upgrade head
 docker-compose down
 ```
 
-The API documentation will be available on http://api.docker.localhost/docs.
+The API documentation will be available on http://api.docker.localhost/docs.  
+By default, the admin user is `[email protected]` and the password is `admin`.
 
 ## Tests
 
@@ -82,4 +83,3 @@ poetry run pre-commit run --all-files
 
 Please use [bumpversion](https://pypi.org/project/bumpversion/0.6.0/) to conduct the releases.
 The version bump will automatically create a new commit associated with a tag.
-When pushed to GitHub, the tag will trigger a deployment workflow that will push the new version of Iris Agent into Docker Hub.
diff --git a/docs/index.md b/docs/index.md
@@ -0,0 +1,11 @@
+# Introduction
+
+Iris is a system to coordinate complex network measurements from multiple vantage points.
+Think of it as a project similar to [CAIDA Ark](https://www.caida.org/projects/ark/) or [RIPE Atlas](https://atlas.ripe.net), with the following features:
+
+- Fully open-source code.
+- Handle multi-round measurements, such as [diamond-miner](https://github.com/dioptra-io/diamond-miner) IP tracing measurements.
+- Handle both centralized computation on a powerful server, and distributed probing on smaller agents.
+- Can tolerate the temporary loss of agents, database or control-plane components.
+
+We offer a public instance of Iris, as well as public measurement data, on [iris.dioptra.io](https://iris.dioptra.io).
diff --git a/iris/__init__.py b/iris/__init__.py
@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "1.1.2"
diff --git a/iris/commons/models/measurement.py b/iris/commons/models/measurement.py
@@ -39,19 +39,25 @@ def check_tool_parameters(cls, values):
         agents: list[MeasurementAgentCreate] = values.get("agents")
         tool: Tool = values.get("tool")
         for agent in agents:
-            if tool in (Tool.DiamondMiner, Tool.Yarrp):
+            if tool == Tool.DiamondMiner:
                 # NOTE: We could use other values, but this would require to change
                 # the Diamond-Miner results schema which has a materialized column
                 # for the destination prefix which assumes /24 and /64 prefixes.
                 if agent.tool_parameters.prefix_len_v4 != 24:
-                    raise ValueError(
-                        "`prefix_len_v4` must be 24 for diamond-miner and yarrp"
-                    )
+                    raise ValueError("`prefix_len_v4` must be 24 for diamond-miner")
                 if agent.tool_parameters.prefix_len_v6 != 64:
-                    raise ValueError(
-                        "`prefix_len_v6` must be 64 for diamond-miner and yarrp"
-                    )
-            if tool in (Tool.Ping,):
+                    raise ValueError("`prefix_len_v6` must be 64 for diamond-miner")
+
+            if tool == Tool.Yarrp:
+                # NOTE: Even though it is possible to use /32 and /128 prefixes
+                # with yarrp, the materialized column for the destination
+                # prefix will be incorrect.
+                if agent.tool_parameters.prefix_len_v4 not in (24, 32):
+                    raise ValueError("`prefix_len_v4` must be 24 or 32 for yarrp")
+                if agent.tool_parameters.prefix_len_v6 not in (64, 128):
+                    raise ValueError("`prefix_len_v6` must be 64 or 128 for yarrp")
+
+            if tool == Tool.Ping:
                 # NOTE: Technically we could use a larger prefix length to allow
                 # the flow mapper to choose a random IP address inside the prefix,
                 # but users probably expect ping to target a specific IP address.