diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0e3e94e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +__pycache__ +venv +deploy.yaml + +data +updata.sh +NOTES.txt +software.yaml diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..400d33b --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,58 @@ +name: docker + +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + +jobs: + docker: + runs-on: ubuntu-latest + permissions: + packages: write + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ncsa/puppet-enc + ghcr.io/${{ github.repository_owner }}/puppet-enc + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5f7cf6e --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__ +venv +.idea + +data +updata.sh +NOTES.txt +software.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..88b0e6b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## 1.0.0 - 2023-04-26 + +Initial release of puppet external node classifier. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..703c970 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM --platform=amd64 python:3.11 + +WORKDIR /app +VOLUME /data +EXPOSE 8080 +ENV PREFIX="/enc" \ + PYTHONUNBUFFERED=1 + +COPY requirements.txt ./ +RUN pip install -r ./requirements.txt + +COPY enc.py ./ +COPY example ./data/ +CMD waitress-serve --url-prefix=${PREFIX} enc:app diff --git a/README.md b/README.md new file mode 100644 index 0000000..ffd3899 --- /dev/null +++ b/README.md @@ -0,0 +1,211 @@ +## Puppet External Node Classifiers + +This repository contains the code for a simple [puppet external node classifier](https://www.puppet.com/docs/puppet/8/nodes_external.html). This classifier uses a REST API to control the classifier, and can easily be added to the puppet server helm deployment. This will allow you to group nodes based on hostname patters, or explicitly on their hostnames. You can add users to managed the default patters, or manage hosts. + +## Using puppet-enc + +You can start the code on your local machine: + +```bash +pip install -r requirements.txt +waitress-server enc.py +``` + +Pr you can use docker: + +```bash +docker run --name enc --volume ${PWD}/enc:/app/data --ports 8080:8080 ncsa/puppet-enc +``` + +In both cases you can access the code at http://localhost:8080 + +Or you can use this with [puppet helm chart](https://github.com/puppetlabs/puppetserver-helm-chart) by adding the following snippet to your values.yaml file (to enable enc) and use the [deploy.yaml](deploy.yaml) file to deploy puppet-enc in the cluster. + +```yaml +puppetserver: + customentrypoints: + enabled: true + configmaps: + 90-custom.sh: | + #!/bin/bash + cat << EOF > /enc.sh + #!/bin/bash + curl -s -u puppet:viewer http://enc:8080/enc/hosts/\$1 + EOF + chmod 755 /enc.sh + /opt/puppetlabs/bin/puppet config set node_terminus "exec" + /opt/puppetlabs/bin/puppet config set external_nodes "/enc.sh" +``` + +## Rest API + +Following is a list of the REST endpoints. They will interact on individual hosts, group of hosts, users and miscalanous actions. All functions require a username and password. Users will have one of the following three roles, and certain actions require certain roles: + +- **admin**: users in this group can do all actions on all objects +- **user**: can modify hosts +- **viewer**: can view hosts + +### HOSTS + +This will return output specified in the [complete example](https://www.puppet.com/docs/puppet/8/nodes_external.html#enc_output_format-section_oxs_qvm_thb) of the puppet documentation. + +```yaml +foo.example.com + classes: + - profile::base + environment: production + parameters: + project: moonshot +``` + +#### *GET /hosts* (**admin**, **user** or **viewer**) + +Return a list of all hosts declared, this will not return hosts specified in the groups. + +```bash +curl -s -u user:user http://localhost:5000/hosts +``` + +#### *GET /hosts/:fqdn* (**admin**, **user** or **viewer**) + +Return the classifier for a single host, or matches a host the group definition. If none is found it will return the default group definition. + +```bash +curl -s -u user:user http://localhost:5000/hosts/example.com +``` + +#### *POST /hosts* (**admin** or **user**) + +Create a new host entry. This requires two fields, the fqdn and the host classifier as a yaml document + +```bash +# if hostname.example.com does not exist this will use the default template +curl -s -o template.yaml http://localhost:5000/host/hostname.example.com +curl -s -u admin:admin http://localhost:5000/hosts -d fqdn=hostname.example.com --data-urlencode data@template.yaml +``` + +#### *PUT /hosts/:fqdn* (**admin**, **user** or **viewer**) + +Updates a host definition. If the key of the property is either `environment` or `classe` it will update that field, otherwise it will asume it is a parameter. If the field is a list you can remove a single entry by prefixing that value with a `-`. + +```bash +curl -X PUT -s -u user:user http://localhost:5000/hosts/example.com -d classes=special -d classes=-old +``` + +#### *DELETE /hosts/:fqdn* (**admin**) + +Deletes a host definition. This will only delete a specific host, not remove the host from a group. + +```bash +curl -X DELETE -s -u user:user http://localhost:5000/hosts/example.com +``` + +### GROUPS + +Groups will allow you to use patterns and multiple hosts to define a classifier. For example this can be used to match all `web` servers, etc. The group definition is almost the same as a host classifier, with the extra field `hosts` which is used to match hosts. The special group definition `default` will be used for hosts that are not matched. All hosts mentioned in the default host are ignored. + +```yaml +default: + classes: + profile::base: + environment: production + hosts: [] + parameters: + project: undefined +``` + +#### *GET /groups* (**admin**, **user** or **viewer**) + +Return a list of all groups declared. + +```bash +curl -s -u user:user http://localhost:5000/groups +``` + +#### *GET /groups/:name* (**admin**, **user** or **viewer**) + +Return the group definition for the specified name. + +```bash +curl -s -u user:user http://localhost:5000/groups/default +``` + +#### *POST /groups* (**admin**) + +Create a new group entry. This requires two fields, the name and the group definition as a yaml document. + +```bash +curl -s -u admin:admin http://localhost:5000/groups -d name=webservers --data-urlencode data@group.yaml +``` + +#### *PUT /groups/:name* (**admin**) + +Updates a group definition. If the key of the property is either `environment`, `classes` or `hosts` it will update that field, otherwise it will asume it is a parameter. If the field is a list you can remove a single entry by prefixing that value with a `-`. + +```bash +curl -X PUT -s -u admin:admin http://localhost:5000/groups/webservers -d classes=webservers +``` + +#### *DELETE /groups/:name* (**admin**) + +Deletes a group definition. It is not possible to delete the `default` group. + +```bash +curl -X DELETE -s -u admin:admin http://localhost:5000/groups/webservers +``` + +### USERS + +Get user, if the user is not an admin they can only get their own info. +```yaml +user: + password: pbkdf2:sha256:260000$EUBvsZZ0H6bTEDH9$f31057490735a2b6fcb4c8dab9e1c08ddb879a8347a3bd39c7ab0a6fdf94ec8a + roles: + - user +``` + +#### *GET /users* (**admin**) + +Return a list of all users declared. + +```bash +curl -s -u admin:admin http://localhost:5000/users +``` + +#### *GET /users/:username* (**admin**, **user** or **viewer**) + +Return the user definition for the specified username. If the user is not an admin, only their own user information can be returned. + +```bash +curl -s -u user:user http://localhost:5000/users/user +``` + +#### *POST /users* (**admin**) + +Create a new user. This requires two fields, the username and the password. It is possible to specify the roles of the user as well, if no roles are specified the user is assigned the role viewer. + +```bash +curl -s -u admin:admin http://localhost:5000/users -d username=bob -d password=foo -d roles=user +``` + +#### *PUT /users/:username* (**admin**) + +Updates a user definition. In th case of roles you can remove a single entry by prefixing that value with a `-`. It is not possible to change the username + +```bash +curl -X PUT -s -u admin:admin http://localhost:5000/users/bob -d roles=admin +``` + +#### *DELETE /users/:username* (**admin**) + +Deletes a group definition. It is not possible to delete the `default` group. + +```bash +curl -X DELETE -s -u admin:admin http://localhost:5000/users/bob +``` + +### MISC + +#### *GET /healtz* + +Used to check if the endpoint is alive. Will always return OK. diff --git a/deploy.yaml b/deploy.yaml new file mode 100644 index 0000000..6c5d2f6 --- /dev/null +++ b/deploy.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-enc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + #storageClassName: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: puppet-enc +spec: + replicas: 1 + selector: + matchLabels: + app: puppet-enc + template: + metadata: + labels: + app: puppet-enc + spec: + initContainers: + - name: copy-data + image: ncsa/puppet-enc + command: + - sh + - '-c' + args: + - >- + cd /app/data; + for f in *.yaml; do + if [ ! -e /data.pvc/$f ]; then + cp $f /data.pvc/$f + fi; + done + volumeMounts: + - name: data + mountPath: /data.pvc/ + containers: + - name: enc + image: ncsa/puppet-enc + ports: + - name: http + containerPort: 8080 + hostPort: 8080 + env: + - name: PREFIX + value: /enc + volumeMounts: + - name: data + mountPath: /app/data + livenessProbe: + httpGet: + path: /enc/healthz + port: http + volumes: + - name: data + persistentVolumeClaim: + claimName: data-enc +--- +apiVersion: v1 +kind: Service +metadata: + name: puppet-enc +spec: + ports: + - name: http + targetPort: http + port: 8080 + selector: + app: puppet-enc +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: puppet-enc +spec: + tls: + - secretName: puppet.example.com + rules: + - host: puppet.example.com + http: + paths: + - path: /enc/ + pathType: ImplementationSpecific + backend: + service: + name: puppet-enc + port: + number: 8080 diff --git a/enc.py b/enc.py new file mode 100644 index 0000000..ee66c5d --- /dev/null +++ b/enc.py @@ -0,0 +1,334 @@ +from flask import Flask, request, abort, Response +from flask_httpauth import HTTPBasicAuth +import os +from time import strftime +from werkzeug.security import generate_password_hash, check_password_hash +import yaml + +_hosts = yaml.safe_load(open("data/hosts.yaml")) +_users = yaml.safe_load(open("data/users.yaml")) +_groups = yaml.safe_load(open("data/groups.yaml")) + +app = Flask(__name__) +auth = HTTPBasicAuth() + + +def make_response(data): + """Create yaml response.""" + resp = Response(response=yaml.dump(data), status=200, mimetype="text/yaml") + # resp.headers['Access-Control-Allow-Origin'] = '*' + return resp + + +def save_data(what, key, value): + """Save value to the file, and return respone with value.""" + if what == "users": + data = _users + elif what == "hosts": + data = _hosts + elif what == "groups": + data = _groups + else: + abort(500) + data = {} + + if value: + data[key] = value + else: + del data[key] + with open(os.path.join("data", f"{what}.yaml"), "w") as fp: + yaml.dump(data, fp) + return make_response(value) + + +@auth.verify_password +def verify_password(username, password): + """Check username/password, return username if found.""" + if username in _users and check_password_hash(_users.get(username)["password"], password): + return username + + +@auth.get_user_roles +def get_user_roles(username): + """Return list of roles for the user""" + if username in _users: + return _users.get(username)["roles"] + + +# ====================================================================== + + +@app.after_request +def after_request(response): + if "/healthz" != request.path: + # 127.0.0.1 - Scott [10/Dec/2019:13:55:36 -0700] "GET /server-status HTTP/1.1" 200 2326 + logline = f"{request.remote_addr} -" + if auth.current_user(): + logline = logline + f" {auth.current_user()}" + else: + logline = logline + " -" + logline = logline + strftime(' [%Y/%b/%d:%H:%M:%S]') + logline = logline + f' "{request.method} {request.path} {request.environ["SERVER_PROTOCOL"]}"' + logline = logline + f" {response.status_code} {response.content_length}" + print(logline) + return response + +# ====================================================================== + + +@app.route("/healthz") +def root(): + return "OK" + + +# ====================================================================== + + +@app.route("/hosts", methods=['GET']) +@auth.login_required(role=['admin', 'user', 'viewer']) +def list_hosts(): + return make_response(list(_hosts.keys())) + + +@app.route("/hosts/", methods=['GET']) +@auth.login_required(role=['admin', 'user', 'viewer']) +def get_host(fqdn): + host = _hosts.get(fqdn) + if host: + return make_response(host) + + for (k, v) in _groups.items(): + if k == "default": + continue + for h in v['hosts']: + if fqdn.startswith(h): + host = v.copy() + del host['hosts'] + return make_response(host) + + v = _groups.get("default") + if v: + host = v.copy() + del host['hosts'] + return make_response(host) + + return make_response(None) + + +@app.route("/hosts", methods=['POST']) +@auth.login_required(role=['admin', 'user']) +def add_host(): + """Add a new host (fqdn) with specified data (yaml format)""" + fqdn = request.form.get("fqdn") + if not fqdn: + abort(400) + data = request.form.get("data") + if not data: + abort(400) + return save_data("hosts", fqdn, yaml.safe_load(data)) + + +@app.route("/hosts/", methods=['PUT']) +@auth.login_required(role=['admin', 'user']) +def update_host(fqdn): + host = _hosts.get(fqdn) + if not host: + abort(404) + for k in request.form.keys(): + if k == "fqdn": + continue + if k == "environment": + host[k] = request.form.get(k) + elif k == "classes": + for v in request.form.getlist(k): + if v.startswith("-"): + if v[:1] in host["classes"]: + del host["classes"][v[:1]] + elif v not in host["classes"]: + host["classes"][v] = None + else: + if isinstance(host["parameters"].get(k, None), list): + for v in request.form.getlist(k): + if v.startswith("-"): + if v[:1] in host["parameters"][k]: + host["parameters"][k].remove(v[:1]) + elif v not in host["parameters"][k]: + host["parameters"][k].append(v) + else: + v = request.form.get(k) + if v.startswith("-"): + if k in host["parameters"] and v[:1] == host["parameters"][k]: + del host["parameters"][k] + else: + host["parameters"][k] = v + + return save_data("hosts", fqdn, host) + + +@app.route("/hosts/", methods=['DELETE']) +@auth.login_required(role=['admin']) +def delete_host(fqdn): + if fqdn not in _hosts: + abort(404) + return save_data("hosts", fqdn, None) + +# ====================================================================== + + +@app.route("/groups", methods=['GET']) +@auth.login_required(role=['admin', 'user', 'viewer']) +def list_groups(): + return make_response(list(_groups.keys())) + + +@app.route("/groups/", methods=['GET']) +@auth.login_required(role=['admin', 'user', 'viewer']) +def get_group(name): + if name not in _groups: + abort(404) + return make_response(_groups.get(name)) + + +@app.route("/groups", methods=['POST']) +@auth.login_required(role=['admin']) +def add_group(): + """Add a new group with name and with specified data (yaml format)""" + name = request.form.get("name") + if not name: + abort(400) + data = request.form.get("data") + if not data: + abort(400) + return save_data("groups", name, yaml.safe_load(data)) + + +@app.route("/groups/", methods=['PUT']) +@auth.login_required(role=['admin']) +def update_group(name): + data = _groups.get(name, {}) + if not data: + abort(404) + + for k in request.form.keys(): + if k == "name": + continue + if k == "environment": + data[k] = request.form.get(k) + elif k == "classes": + for v in request.form.getlist(k): + if v.startswith("-"): + if v[:1] in data["classes"]: + del data["classes"][v[:1]] + elif v not in data["classes"]: + data["classes"][v] = None + elif k == "hosts": + for v in request.form.getlist(k): + if v.startswith("-"): + if v[:1] in data["hosts"]: + data["hosts"].remove(v[:1]) + elif v not in data["hosts"]: + data["hosts"].append(v) + else: + if isinstance(data["parameters"].get(k, None), list): + for v in request.form.getlist(k): + if v.startswith("-"): + if v[:1] in data["parameters"][k]: + data["parameters"][k].remove(v[:1]) + elif v not in data["parameters"][k]: + data["parameters"][k].append(v) + else: + v = request.form.get(k) + if v.startswith("-"): + if k in data["parameters"] and v[:1] == data["parameters"][k]: + del data["parameters"][k] + else: + data["parameters"][k] = v + + return save_data("groups", name, data) + + +@app.route("/groups/", methods=['DELETE']) +@auth.login_required(role=['admin']) +def delete_group(name): + if name not in _groups: + abort(404) + if name == "default" + abort(403) + return save_data("users", name, None) + +# ====================================================================== + + +@app.route("/users", methods=['GET']) +@auth.login_required(role=['admin']) +def list_users(): + return make_response(list(_users.keys())) + + +@app.route("/users/", methods=['GET']) +@auth.login_required +def get_user(username): + if username != auth.current_user() and "admin" not in get_user_roles(auth.current_user()): + abort(403) + if username not in _users: + abort(404) + return make_response(_users.get(username)) + + +@app.route("/users", methods=['POST']) +@auth.login_required(role=['admin']) +def add_user(): + username = request.form.get("username") + if not username: + abort(400) + if username in _users: + abort(400) + password = request.form.get("password") + if not password: + abort(400) + password = generate_password_hash(password) + roles = request.form.getlist("role") + if not roles: + roles = ["viewer"] + user = {"password": password, "roles": roles} + return save_data("users", username, user) + + +@app.route("/users/", methods=['PUT']) +@auth.login_required(role=['admin']) +def update_user(username): + user = _users.get(username, {}) + if not user: + abort(404) + + password = request.form.get("password") + if password: + user['password'] = generate_password_hash(password) + for v in request.form.getlist("roles"): + if v.startswith("-"): + if v[:1] in user["roles"]: + user["roles"].remove(v[:1]) + elif v not in user["roles"]: + user["roles"].append(v) + return save_data('users', username, user) + + +@app.route("/users/", methods=['DELETE']) +@auth.login_required(role=['admin']) +def delete_user(username): + if username not in _users: + abort(404) + return save_data("users", username, None) + +# ====================================================================== + + +def represent_none(self, _): + return self.represent_scalar('tag:yaml.org,2002:null', '') + + +# set yaml.dump to print empty value for None +yaml.add_representer(type(None), represent_none) + +if __name__ == '__main__': + app.run(debug=True) diff --git a/example/groups.yaml b/example/groups.yaml new file mode 100644 index 0000000..3acce5b --- /dev/null +++ b/example/groups.yaml @@ -0,0 +1,7 @@ +default: + classes: + profile::base: + environment: production + hosts: [] + parameters: + project: undefined diff --git a/example/hosts.yaml b/example/hosts.yaml new file mode 100644 index 0000000..1eb1875 --- /dev/null +++ b/example/hosts.yaml @@ -0,0 +1,2 @@ +{ } + diff --git a/example/users.yaml b/example/users.yaml new file mode 100644 index 0000000..fff427c --- /dev/null +++ b/example/users.yaml @@ -0,0 +1,12 @@ +admin: + password: pbkdf2:sha256:260000$t2Ks8rp69wPx2OF3$630363a58db1a18da2542008e4bcfcd2876e476de39cb7ba334fd744a6d2b0e2 + roles: + - admin +user: + password: pbkdf2:sha256:260000$EUBvsZZ0H6bTEDH9$f31057490735a2b6fcb4c8dab9e1c08ddb879a8347a3bd39c7ab0a6fdf94ec8a + roles: + - user +puppet: + password: pbkdf2:sha256:260000$h92toOoGjMJrDNs0$b1da21420a141c8133917e3a3a3933718d3940be8f2f39b64580ccb913ac1010 + roles: + - viewer diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4954d5b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +flask +waitress +pyyaml +flask_httpauth