diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..2ac45c3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,38 @@ +image: docker:stable + +services: + - docker:dind + +variables: + DOCKER_DRIVER: overlay2 + +stages: + - deploy + + +deploy_to_docker_hub: + stage: deploy + when: manual + only: + # We only want master branch AND when tag looks like 'vX.Y.Z', however GitLab doesn't support conjunctive conditions yet: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/27818 + # refs: + # - master # Yeah, that doesn't work... The job for a commit with a tag and on a master branch is not being created. + # + # However we can mark tags 'v*.*.*' as protected, which also allows us to (somewhat) safely use Private-Token as protected + # CI variable. + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ + script: + - apk add --no-cache git + # check that we are deploying the latest version: + - export LAST_KNOWN_VERSION=`git tag -l --sort=-version:refname "v*.*.*" | head -n 1 | tr -d '[:space:]'` + - '[ "$LAST_KNOWN_VERSION" == "$CI_COMMIT_TAG" ] || (echo "Tag does not denote latest known version (which is $LAST_KNOWN_VERSION), aborting!" && exit 1)' + - echo "Deploying..." + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$CI_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" + - docker push "$CI_REGISTRY_IMAGE:latest" + - docker rmi grafolean/grafolean:$CI_COMMIT_TAG + - docker rmi grafolean/grafolean:latest + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f073592 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.6-alpine + +COPY requirements.txt grafolean-worker-ping.py /opt/ +RUN pip install -r /opt/requirements.txt +RUN echo -e "* * * * * source /etc/environment; export BACKEND_URL BOT_TOKEN; python /opt/grafolean-worker-ping.py > /proc/1/fd/1 2> /proc/1/fd/2\n" > /etc/crontabs/root + +# https://stackoverflow.com/a/47960145/593487 +ENTRYPOINT ["crond", "-f", "-d", "8"] diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..2c394f9 --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +multiping = "*" + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..ca80087 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,66 @@ +{ + "_meta": { + "hash": { + "sha256": "98beea604639596f2305d194054f2495ce6a259c656b23cef40878f45938b1b8" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "multiping": { + "hashes": [ + "sha256:8bad44be68db553efedc46c6af51a49660b19e6731bfe53311dddfdf30397b25", + "sha256:916bc870ee4e4921ad9dbf997a15e046195480a775ec7c2c615f4de9a5fd3483", + "sha256:9ecad777ca27c4415cd685957bafaaa0faf4f17f9419800e9f57a298201f2e5d" + ], + "index": "pypi", + "version": "==1.1.2" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + } + }, + "develop": {} +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6224564 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '2.1' +services: + + # To run this image, you must create a .env file with following content: + # BACKEND_URL=... # backend API base url (the part before "/api/...") + # BOT_TOKEN=... # bot token + + grafolean-worker-ping: + # If you wish to load an explicit version, change the next line. For example: + # image: grafolean/grafolean-worker-ping:v1.0.0 + image: grafolean/grafolean-worker-ping + build: + context: . + dockerfile: Dockerfile + volumes: + - .env:/etc/environment diff --git a/grafolean-worker-ping.py b/grafolean-worker-ping.py new file mode 100644 index 0000000..949b4b7 --- /dev/null +++ b/grafolean-worker-ping.py @@ -0,0 +1,87 @@ +from collections import defaultdict +from multiping import MultiPing +import os +import requests +import socket +import time + +N_PINGS = 3 + +# This is copy-pasted from multiping package; the reason is that we need to get MultiPing +# instance, because it holds the IP addresses which correspond to the addresses we wanted +# pinged - and the key in ping results is the IP. +def multi_ping(dest_addrs, timeout, retry=0, ignore_lookup_errors=False): + retry_timeout = float(timeout) / (retry + 1) + + mp = MultiPing(dest_addrs, ignore_lookup_errors=ignore_lookup_errors) + + results = {} + retry_count = 0 + while retry_count <= retry: + # Send a batch of pings + mp.send() + single_results, no_results = mp.receive(retry_timeout) + # Add the results from the last sending of pings to the overall results + results.update(single_results) + if not no_results: + # No addresses left? We are done. + break + retry_count += 1 + + return results, no_results, mp + +def get_addr_for_ip_dict(addrs, mp): + # This is complicated, and a hack. Still... mp (MultiPing instance) holds two lists, + # self._dest_addrs and self._unprocessed_targets. List _unprocessed_targets has the addresses + # that couldn't be resolved. Others were resolved, and _dest_addrs has the IPs in the same + # order as original addresses. + resolved_addrs = [a for a in addrs if a not in mp._unprocessed_targets] + ip_to_addr = {k: v for k, v in zip(mp._dest_addrs, resolved_addrs)} + return ip_to_addr + +def do_ping(addrs): + results = defaultdict(list) + mp = None + ip_to_addr = {} + for i in range(N_PINGS): + print(".") + responses, no_responses, mp = multi_ping(addrs, timeout=2, retry=3, ignore_lookup_errors=True) + + # Some addresses (like demo.grafolean.com) resolve to multiple IPs, so each call to multi_ping will + # resolve differently - we must find the new IP addresses every time: + ip_to_addr = get_addr_for_ip_dict(addrs, mp) + + for no_resp in no_responses: + addr = ip_to_addr.get(no_resp, no_resp) + results[addr].append(None) + for resp, t in responses.items(): + addr = ip_to_addr.get(resp, resp) + results[addr].append(t) + + if i < N_PINGS - 1: + time.sleep(1) + return dict(results) + +def send_results_to_grafolean(base_url, account_id, bot_token, results): + url = '{}/api/accounts/{}/values/?b={}'.format(base_url, account_id, bot_token) + values = [] + for ip in results: + for ping_index, ping_time in enumerate(results[ip]): + values.append({ + 'p': 'ping.{}.{}.success'.format(ip.replace('.', '_'), ping_index), + 'v': 0 if ping_time is None else 1, + }) + if ping_time is not None: + values.append({ + 'p': 'ping.{}.{}.rtt'.format(ip.replace('.', '_'), ping_index), + 'v': ping_time, + }) + print("Sending results to Grafolean") + r = requests.post(url, json=values) + print(r.text) + r.raise_for_status() + +if __name__ == "__main__": + addrs = ["8.8.8.8", "youtube.com", "127.0.0.1", "demo.grafolean.com", "grafolean.com", "whateverdoeesndfexist.com"] + results = do_ping(addrs) + send_results_to_grafolean(os.environ.get('BACKEND_URL'), 1, os.environ.get('BOT_TOKEN'), results) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..186c1ca --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +-i https://pypi.org/simple +certifi==2018.11.29 +chardet==3.0.4 +idna==2.8 +multiping==1.1.2 +requests==2.21.0 +urllib3==1.24.1