Merge pull request #8 from palto42/develop

Version 0.6.1

.travis.yml

@@ -12,6 +12,7 @@ python:
   # - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
 
 install:
   - make init

.vscode/settings.json

@@ -2,27 +2,44 @@
     "python.linting.flake8Enabled": true,
     "python.linting.enabled": true,
     "cSpell.words": [
+        "CACERTDIR",
+        "CACERTFILE",
         "PYTHONPATH",
         "Postgres",
+        "TABLENAMES",
         "apscheduler",
         "apscheduler's",
+        "auditlogs",
         "bcrypt",
+        "bdist",
         "checkpw",
         "corescheduler",
+        "dateutil",
         "dockerized",
+        "funcsigs",
         "hashpw",
         "htpasswd",
         "ioloop",
+        "jobauditlog",
         "keyout",
+        "ldaps",
         "mrkdwn",
         "ndscheduler",
         "newkey",
         "palto",
+        "pypi",
         "rtype",
         "sched",
+        "sdist",
         "selfsigned",
         "sendgrid",
+        "sqlite",
+        "sslmode",
+        "tablename",
         "urlencode",
-        "venv"
-    ]
+        "venv",
+        "webcron",
+        "wheel"
+    ],
+    "python.pythonPath": ".venv/bin/python"
 }

Makefile

@@ -19,7 +19,7 @@ test:
 	make install
 	make flake8
 	# Hacky way to ensure mock is installed before running setup.py
-	$(SOURCE_VENV) && pip install mock==1.1.2 && $(PYTHON) setup.py test
+	$(SOURCE_VENV) && pip install -r test_requirements.txt && $(PYTHON) setup.py test
 
 install:
 	make init

README.md

@@ -1,7 +1,7 @@
 # Nextdoor Scheduler
 
 [![License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg)](LICENSE.txt)
-[![Build Status](https://api.travis-ci.org/palto42/ndscheduler.svg)](https://travis-ci.org/palto42/ndscheduler)
+[![Build Status](https://api.travis-ci.com/palto42/ndscheduler.svg)](https://travis-ci.com/palto42/ndscheduler)
 
 ``ndscheduler`` is a flexible python library for building your own cron-like system to schedule jobs, which is to run a tornado process to serve REST APIs and a web ui.
 
@@ -42,10 +42,11 @@ Note: ``corescheduler`` can also be used independently within your own service i
     * pip install .
     * Install scheduler implementation like [simple_scheduler](https://github.com/palto42/simple_scheduler)
 3. Configure ~/.config/ndscheduler/config.yaml
-    * See [example configuration](config_example.yaml)
-    * Passwords must be hashed with bcrypt
-    * See [Python bcrypt tutorial](http://zetcode.com/python/bcrypt/)
-    * More ideas for basic_auth [Tornado basic auth example](https://gist.github.com/notsobad/5771635)
+    * See [example configuration](config_example.yaml) and [default configuration](ndscheduler/config_default.yaml) for available options.
+    * Optionally, enable authentication
+      * For local authentication, configure users and passwords as in the [example configuration](config_example.yaml). Passwords must be hashed with bcrypt, which can be done with the command `python -m ndscheduler --encrypt`
+      * For LDAP authentication, configure the LDAP server settings and the list of allowed users.
+        * If LDAP authentication should be used, the Python package `python-ldap`must be installed.
 4. Start scheduler implementation
 5. Launch web browser at configured URL and authenticate with configured account
 

config_example.yaml

@@ -18,7 +18,7 @@ SSL_KEY: /path/to/key  # required for HTTPS
 #     "jobs_tablename": confuse.String(default="scheduler_jobs"),
 #     "executions_tablename": confuse.String(default="scheduler_execution"),
 #     "auditlogs_tablename": confuse.String(default="scheduler_jobauditlog"),
-# DATABASE_CLASS: sqlite  # supporte: sqlite, postgres, mysql
+# DATABASE_CLASS: sqlite  # supported: sqlite, postgres, mysql
 DATABASE_CONFIG_DICT:
     "file_path": datastore.db  # SQlite
     # additional attributes for MySQL and Postgres

ndscheduler/__init__.py

@@ -21,6 +21,7 @@
 import bcrypt
 from getpass import getpass
 from time import sleep
+import pkg_resources
 
 from ndscheduler import default_settings
 
@@ -85,6 +86,13 @@ def get_cli_args():
     parser.add_argument(
         "--encrypt", "-e", help="Create hash value from password for use in AUTH_CREDENTIALS.", action="store_true",
     )
+    parser.add_argument(
+        "--version",
+        "-V",
+        action="version",
+        help="Show version",
+        version=f"%(prog)s fla{pkg_resources.get_distribution('construct').version}",
+    )
 
     args, _ = parser.parse_known_args()
 
@@ -186,6 +194,16 @@ def load_yaml_config(
         "MAIL_SERVER": confuse.StrSeq(),
         "ADMIN_MAIL": confuse.StrSeq(),
         "SERVER_MAIL": confuse.String(default=""),
+        # LDAP server addess in the format "ldap://my.ldap.server" "ldaps://my.ldap.server"
+        # Non-standard ports can be specified like "ldap://my.ldap.server:1234"
+        "LDAP_SERVER": confuse.String(default=""),
+        "LDAP_REQUIRE_CERT": confuse.Choice(["demand", "allow", "never"], default="demand",),
+        "LDAP_CERT_DIR": confuse.String(default=None),
+        "LDAP_CERT_FILE": confuse.String(default=None),
+        # Define LDAP dn format for login, {username} will be replaced with the entered user name
+        "LDAP_LOGIN_DN": confuse.String(default="uid={username},ou=people,o=MyCompany,dc=net"),
+        # List of permitted LDAP users. If none are specified, any authenticated used is allowed
+        "LDAP_USERS": confuse.StrSeq(default=[]),
     }
 
     yaml_template.update(yaml_extras)

ndscheduler/config_default.yaml

@@ -85,3 +85,14 @@ MAIL_SERVER: []
 
 # Server sender mail address
 SERVER_MAIL: ""
+
+# LDAP server address in the format "ldap://my.ldap.server" "ldaps://my.ldap.server"
+# Non-standard ports can be specified like "ldap://my.ldap.server:1234"
+# LDAP_SERVER: ldaps://my.ldap.server
+LDAP_REQUIRE_CERT: demand
+# LDAP_CERT_DIR: None
+# LDAP_CERT_FILE: None
+# Define LDAP dn format for login, {username} will be replaced with the entered user name
+LDAP_LOGIN_DN: uid={username},ou=users,dc=example,dc=com
+# List of permitted LDAP users. If none are specified, any authenticated used is allowed
+LDAP_USERS: []

ndscheduler/corescheduler/core/base.py

@@ -52,10 +52,7 @@ def run_job(
         execution_id = utils.generate_uuid()
         datastore = utils.get_datastore_instance(db_class_path, db_config, db_tablenames)
         datastore.add_execution(
-            execution_id,
-            job_id,
-            constants.EXECUTION_STATUS_SCHEDULED,
-            description=JobBase.get_scheduled_description(),
+            execution_id, job_id, constants.EXECUTION_STATUS_SCHEDULED, description=JobBase.get_scheduled_description(),
         )
         try:
             job_class = utils.import_from_path(job_class_path)

ndscheduler/corescheduler/datastore/base_test.py

@@ -41,16 +41,10 @@ def test_get_executions_by_time_interval(self):
             "12", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=5),
         )
         self.store.add_execution(
-            "13",
-            "34",
-            state=constants.EXECUTION_STATUS_SCHEDULED,
-            scheduled_time=now + datetime.timedelta(minutes=50),
+            "13", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=50),
         )
         self.store.add_execution(
-            "14",
-            "34",
-            state=constants.EXECUTION_STATUS_SCHEDULED,
-            scheduled_time=now + datetime.timedelta(minutes=70),
+            "14", "34", state=constants.EXECUTION_STATUS_SCHEDULED, scheduled_time=now + datetime.timedelta(minutes=70),
         )
         self.store.add_execution(
             "15",

ndscheduler/default_settings.py

@@ -118,7 +118,6 @@
     # "user": "$2y$11$MCw3cm9Tp.8zF/hmPILW3.1hGMtP0UV8kUevfaxrzM7JzXdoyFi6.",  # Very$ecret
 }
 
-
 # List of admin users
 ADMIN_USER = []
 
@@ -130,3 +129,16 @@
 
 # Server sender mail address
 SERVER_MAIL = ""
+
+# LDAP server address in the format "ldap://my.ldap.server" "ldaps://my.ldap.server"
+# Non-standard ports can be specified like "ldap://my.ldap.server:1234"
+LDAP_SERVER = ""
+# If "ldaps://" is used, specify of the SSL certificate should be verified
+# Possible options are "demand", "allow" or "never"
+LDAP_REQUIRE_CERT = "demand"
+LDAP_CERT_DIR = None
+LDAP_CERT_File = None
+# Define LDAP dn format for login, {username} will be replaced with the entered user name
+LDAP_LOGIN_DN = "uid={username},ou=users,dc=example,dc=com"
+# List of permitted LDAP users. If none are specified, any authenticated used is allowed
+LDAP_USERS = []

ndscheduler/server/handlers/audit_logs.py

@@ -57,8 +57,9 @@ def get(self):
                             "event": "modified",
                             "user": "",
                             "created_time": "",
-                            "description": ("<script>$('#modalLoginForm').modal"
-                                            "({backdrop: 'static', keyboard: false});</script>"),
+                            "description": (
+                                "<script>$('#modalLoginForm').modal" "({backdrop: 'static', keyboard: false});</script>"
+                            ),
                         }
                     ]
                 }

ndscheduler/server/handlers/base.py

@@ -7,6 +7,7 @@
 import logging
 import json
 import bcrypt
+import ldap
 
 from concurrent import futures
 
@@ -70,18 +71,75 @@ def get(self):
 
     def post(self):
         username = self.get_argument("username")
+        password = self.get_argument("password")
         hashed = self.auth_credentials.get(username)
         logger.debug(f"Received login for user '{username}'")
-        if hashed is not None and bcrypt.checkpw(self.get_argument("password").encode(), hashed.encode()):
-            # 6h = 0.25 days
-            # 1h = 0.041666667 days
-            # 1min = 0.000694444 days
-            self.set_secure_cookie(settings.COOKIE_NAME, username, expires_days=settings.COOKIE_MAX_AGE)
-            logger.debug(f"Set cookie for user {username}, expires: {settings.COOKIE_MAX_AGE * 1440} minutes")
-            self.redirect("/")
+        if settings.LDAP_SERVER and self.ldap_login(username, password):
+            self.set_user_cookie(username)
+        elif hashed is not None and bcrypt.checkpw(password.encode(), hashed.encode()):
+            logger.debug("Try local authentication")
+            self.set_user_cookie(username)
         else:
             logger.debug("Wrong username or password")
-            self.redirect("/")
+        self.redirect("/")
+
+    def set_user_cookie(self, username):
+        # 6h = 0.25 days
+        # 1h = 0.041666667 days
+        # 1min = 0.000694444 days
+        self.set_secure_cookie(settings.COOKIE_NAME, username, expires_days=settings.COOKIE_MAX_AGE)
+        logger.debug(f"Set cookie for user {username}, expires: {settings.COOKIE_MAX_AGE * 1440} minutes")
+
+    def ldap_login(self, username, password):
+        """Verifies credentials for username and password.
+
+        Parameters
+        ----------
+        username : str
+            User ID (uid) to be used for login
+        password : str
+            User password
+
+        Returns
+        -------
+        bool
+            True if login was successful
+        """
+        if settings.LDAP_USERS and username not in settings.LDAP_USERS:
+            logging.warning(f"User {username} not allowed for LDAP login")
+            return False
+        LDAP_SERVER = settings.LDAP_SERVER
+        # Create fully qualified DN for user
+        LDAP_DN = settings.LDAP_LOGIN_DN.replace("{username}", username)
+        logger.debug(f"LDAP dn: {LDAP_DN}")
+        # disable certificate check
+        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
+
+        # specify certificate dir or file
+        if settings.LDAP_CERT_DIR:
+            ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, settings.LDAP_CERT_DIR)
+        if settings.LDAP_CERT_FILE:
+            ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, settings.LDAP_CERT_FILE)
+        try:
+            # build a client
+            ldap_client = ldap.initialize(LDAP_SERVER)
+            ldap_client.set_option(ldap.OPT_REFERRALS, 0)
+            # perform a synchronous bind to test authentication
+            ldap_client.simple_bind_s(LDAP_DN, password)
+            logger.info(f"User '{username}' successfully authenticated via LDAP")
+            ldap_client.unbind_s()
+            return True
+        except (ldap.INVALID_CREDENTIALS, ldap.NO_SUCH_OBJECT):
+            ldap_client.unbind()
+            logger.warning("LDAP: wrong username or password")
+        except ldap.SERVER_DOWN:
+            logger.warning("LDAP server not available")
+        except ldap.LDAPError as e:
+            if isinstance(e, dict) and "desc" in e:
+                logger.warning(f"LDAP error: {e['desc']}")
+            else:
+                logger.warning(f"LDAP error: {e}")
+        return False
 
 
 class LogoutHandler(BaseHandler):

ndscheduler/server/handlers/jobs.py

@@ -329,6 +329,5 @@ def _validate_post_data(self):
 
         if not valid_cron_string:
             raise tornado.web.HTTPError(
-                400,
-                reason=("Require at least one of following parameters:" " %s" % str(at_least_one_required_fields)),
+                400, reason=("Require at least one of following parameters:" " %s" % str(at_least_one_required_fields)),
             )

ndscheduler/version.py

@@ -1 +1 @@
-__version__ = "0.6.0"  # http://semver.org/
+__version__ = "0.6.1"  # http://semver.org/

setup.py

@@ -12,7 +12,7 @@
 
 multiprocessing
 
-PACKAGE = "ndscheduler"
+PACKAGE = "ndscheduler-fork"
 __version__ = None
 
 exec(open(os.path.join("ndscheduler", "version.py")).read())  # set __version__
@@ -61,10 +61,10 @@ def maybe_rm(path):
     version=__version__,
     description="ndscheduler: A cron-replacement library from Nextdoor",
     long_description=open("README.md").read(),
-    author="Nextdoor Engineering",
-    author_email="[email protected]",
-    url="https://github.com/Nextdoor/ndscheduler",
-    license="Apache License, Version 2",
+    author="Matthias Homann (original: Nextdoor Engineering)",
+    author_email="[email protected]",
+    url="https://github.com/palto42/ndscheduler",
+    license="BSD 2-Clause 'Simplified' License",
     keywords="scheduler nextdoor cron python",
     packages=find_packages(),
     include_package_data=True,
@@ -79,6 +79,8 @@ def maybe_rm(path):
         "python-dateutil >= 2.2",
         "bcrypt >= 3.1.7",  # for user authentication
         "confuse >= 1.1.0",  # for yaml config support
+        # python-ldap is only required if LDAP authentication is used
+        # "python-ldap >= 3.3.1",
     ],
     classifiers=classifiers,
     cmdclass={"clean": CleanHook},

test_requirements.txt

@@ -0,0 +1,3 @@
+mock==1.1.2
+construct>=2.10
+python-ldap >= "3.3.1"