From 0fc93a59a897de1bc94c1fdc2ff4cc3351e72039 Mon Sep 17 00:00:00 2001 From: adrianweetman Date: Thu, 17 Aug 2023 10:14:13 +0100 Subject: [PATCH 01/29] Add initial python for updating home areas --- cli/__init__.py | 21 ++++++++++----- cli/ldap/update_user_home_areas.py | 43 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 cli/ldap/update_user_home_areas.py diff --git a/cli/__init__.py b/cli/__init__.py index fca6d0f..a278dfd 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,5 +1,5 @@ import click -from cli import ldap +import cli.ldap.update_user_home_areas from cli import git @@ -7,6 +7,12 @@ def main_group(): pass +# Add git test +@click.command() +def git_test(): + git.dl_test() + +# Add roles to username @click.command() @click.option("--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") @click.option("--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") @@ -14,15 +20,16 @@ def main_group(): def add_roles_to_users(user_ou, root_dn, user_role_list): ldap.process_user_roles_list(user_role_list, user_ou, root_dn) - +# Update user home area @click.command() -def git_test(): - git.dl_test() - -# from cli.ldap import test +@click.option("--old-home-area", help="name of old home area") +@click.option("--new-home-area", help="name of new home area") +def update_user_home_areas(old_home_area, new_home_area): + cli.ldap.update_user_home_areas.update_user_home_areas(old_home_area, new_home_area) -main_group.add_command(add_roles_to_users) main_group.add_command(git_test) +main_group.add_command(add_roles_to_users) +main_group.add_command(update_user_home_areas) if __name__ == "__main__": main_group() diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py new file mode 100644 index 0000000..cded374 --- /dev/null +++ b/cli/ldap/update_user_home_areas.py @@ -0,0 +1,43 @@ +import logging + +from cli import config +from cli.ldap import ldap_connect + + +## +# Original ansible to replicate then remove +## + # - name: Search for matching users (Home Area=Old value, Active) + # shell: > + # ldapsearch -LLL -H ldap:// -D cn=root,dc=moj,dc=com -w "${ldap_admin_password}" -b {{ user_dn }} -s one \ + # '(&(objectclass=NDUser)(userHomeArea={{ old_home_area }})(!(cn={{ old_home_area }}))(!(endDate=*)))' dn \ + # | sed 's/^$/changetype: modify\nreplace: userHomeArea\nuserHomeArea: {{ new_home_area }}\n/' \ + # > {{ workspace }}/update.ldif + # environment: + # ldap_admin_password: '{{ ldap_admin_password.stdout }}' + + # - name: Create additional LDIF for deleting home areas (required due to LDAP issue causing old values to get "stuck") + # shell: > + # cat {{ workspace }}/update.ldif \ + # | sed 's/replace: userHomeArea/delete: userHomeArea/' \ + # | sed 's/userHomeArea: {{ new_home_area }}/userHomeArea: {{ old_home_area }}/' \ + # > {{ workspace }}/delete.ldif + + # - name: Apply changes + # shell: ldapmodify -Y EXTERNAL -H ldapi:// -c -f {{ workspace }}/update.ldif 2>&1 | tee {{ workspace }}/update.log + + # - name: Restart LDAP (required due to LDAP issue causing old values to get "stuck") + # shell: systemctl restart slapd + + # - name: Force deletion of old home area (required due to LDAP issue causing old values to get "stuck") + # shell: ldapmodify -Y EXTERNAL -H ldapi:// -c -f {{ workspace }}/delete.ldif 2>&1 | tee {{ workspace }}/delete.log + + +def update_user_home_areas(old_home_area, new_home_area): + logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") + ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) + ldap_connection.search("ou=Users,dc=moj,dc=com", f"(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))") + records = ldap_connection.response + logging.info(len(records)) + for record in records: + logging.info(record) From c6c5ed037f98947923e4fa8e6c96881b81b3c8ba Mon Sep 17 00:00:00 2001 From: adrianweetman Date: Thu, 17 Aug 2023 17:16:30 +0100 Subject: [PATCH 02/29] adding comments for future work --- cli/ldap/update_user_home_areas.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py index cded374..a4d1ed8 100644 --- a/cli/ldap/update_user_home_areas.py +++ b/cli/ldap/update_user_home_areas.py @@ -37,7 +37,12 @@ def update_user_home_areas(old_home_area, new_home_area): logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) ldap_connection.search("ou=Users,dc=moj,dc=com", f"(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))") + + # Output records for debug purposes records = ldap_connection.response logging.info(len(records)) for record in records: logging.info(record) + + # Enter code here modify and delete LDAP entries as per the outcome of the native ansible-invoked LDAP commands above + # But using the python package methods rather than constructing LDIFs From 53c103ce23d28aa42b4d7010d7fd940c6a196082 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:17:06 +0100 Subject: [PATCH 03/29] Update rbac.py --- cli/ldap/rbac.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 9aecaa3..8c666b2 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -226,14 +226,13 @@ def user_ldifs(rendered_files): raise Exception(f"Failed to add user {dn}, status: {ldap_connection.result['result']}") -def main(rbac_repo_tag): +def main(rbac_repo_tag, clone_path="./rbac"): repo = get_repo(rbac_repo_tag) print(env.vars.get("RBAC_SUBSTITUTIONS")) - dir = "./rbac" files = [ file - for file in glob.glob(f"{dir}/**/*", recursive=True) + for file in glob.glob(f"{clone_path}/**/*", recursive=True) if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") ] @@ -245,12 +244,3 @@ def main(rbac_repo_tag): role_ldifs(rendered_files) group_ldifs(rendered_files) user_ldifs(rendered_files) - - parser = LDIFParser(open("./rendered/rbac/context.ldif", "rb"), strict=False) - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - print(record) - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - ldap_connection.add(dn, attributes=record) From 84809e69fe2d74f004a02a6dbbd5f88cfcde668c Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:19:33 +0100 Subject: [PATCH 04/29] pre=release --- .github/workflows/tag-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 6ba5b4c..781e98a 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -34,6 +34,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true PRE_RELEASE_NAME: dev + PRE_RELEASE_BRANCHES: dev - name: release or prerelease id: release_type run: | From ec331aa6971f4d7c3bfe549c7d1ed16ceb999897 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:30:34 +0100 Subject: [PATCH 05/29] prerelease test --- .github/workflows/tag-and-release.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 781e98a..f9415b9 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -27,24 +27,24 @@ jobs: with: ref: ${{ github.event.pull_request.merge_commit_sha }} fetch-depth: '0' - - name: Bump version and push tag - id: tag - uses: anothrNick/github-tag-action@1.67.0 # Don't use @master or @v1 unless you're happy to test the latest version - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - PRE_RELEASE_NAME: dev - PRE_RELEASE_BRANCHES: dev - name: release or prerelease id: release_type run: | - if [[ ${{ steps.tag.outputs.new_tag }} == *"dev"* ]]; then - echo "This is a prerelease" - echo "DEV=true" >> $GITHUB_OUTPUT - else + if [[ ${{github.event.pull_request.base.ref == 'main'}} ]]; then echo "This is a release" echo "DEV=false" >> $GITHUB_OUTPUT + else + echo "This is a prerelease" + echo "DEV=true" >> $GITHUB_OUTPUT fi + - name: Bump version and push tag + id: tag + uses: anothrNick/github-tag-action@1.67.0 # Don't use @master or @v1 unless you're happy to test the latest version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + PRERELEASE_SUFFIX: dev + PRERELEASE: "${{ steps.release_type.outputs.DEV }}" - name: Create prerelease if: steps.release_type.outputs.DEV == 'true' env: From 14099e594042277f90edd70f9d210df963f3a515 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:35:46 +0100 Subject: [PATCH 06/29] PRERELEASE --- .github/workflows/tag-and-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index f9415b9..0ab0798 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -32,10 +32,10 @@ jobs: run: | if [[ ${{github.event.pull_request.base.ref == 'main'}} ]]; then echo "This is a release" - echo "DEV=false" >> $GITHUB_OUTPUT + echo "PRERELEASE=false" >> $GITHUB_OUTPUT else echo "This is a prerelease" - echo "DEV=true" >> $GITHUB_OUTPUT + echo "PRERELEASE=true" >> $GITHUB_OUTPUT fi - name: Bump version and push tag id: tag @@ -44,15 +44,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true PRERELEASE_SUFFIX: dev - PRERELEASE: "${{ steps.release_type.outputs.DEV }}" + PRERELEASE: "${{ steps.release_type.outputs.PRERELEASE }}" - name: Create prerelease - if: steps.release_type.outputs.DEV == 'true' + if: steps.release_type.outputs.PRERELEASE == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh release create ${{ steps.tag.outputs.new_tag }} --title "Dev ${{ steps.tag.outputs.new_tag }}" --prerelease --generate-notes --verify-tag - name: Create release - if: steps.release_type.outputs.DEV == 'false' + if: steps.release_type.outputs.PRERELEASE == 'false' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | From fcad74034aa92a9388c2a12355dfbdb98be4ef6e Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:37:19 +0100 Subject: [PATCH 07/29] release work flow test --- cli/ldap/rbac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 8c666b2..bdad976 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -240,7 +240,7 @@ def main(rbac_repo_tag, clone_path="./rbac"): rendered_files = template_rbac(files) context_ldif(rendered_files) policy_ldifs(rendered_files) - # schema_ldifs(files) probably not needed, but check! + # schema_ldifs(files) probably not needed, but need to check! role_ldifs(rendered_files) group_ldifs(rendered_files) user_ldifs(rendered_files) From e3879faa2b75d2cb5ea4aae0a7cc626b01432841 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:41:17 +0100 Subject: [PATCH 08/29] pre release --- .github/workflows/tag-and-release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 0ab0798..f1d1b2a 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -30,13 +30,13 @@ jobs: - name: release or prerelease id: release_type run: | - if [[ ${{github.event.pull_request.base.ref == 'main'}} ]]; then - echo "This is a release" - echo "PRERELEASE=false" >> $GITHUB_OUTPUT - else - echo "This is a prerelease" - echo "PRERELEASE=true" >> $GITHUB_OUTPUT - fi + if [[ "${{ github.event.pull_request.base.ref }}" == "main" ]]; then + echo "This is a release" + echo "PRERELEASE=false" >> $GITHUB_OUTPUT + else + echo "This is a prerelease" + echo "PRERELEASE=true" >> $GITHUB_OUTPUT + fi - name: Bump version and push tag id: tag uses: anothrNick/github-tag-action@1.67.0 # Don't use @master or @v1 unless you're happy to test the latest version From f3ba40761120699c8d624ce1b117c18df4042d65 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 18 Aug 2023 15:45:04 +0100 Subject: [PATCH 09/29] Update rbac.py --- cli/ldap/rbac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index bdad976..c942a1f 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -143,7 +143,7 @@ def role_ldifs(rendered_files): ldap_connection.delete("cn=ndRoleCatalogue," + env.vars.get("LDAP_CONFIG").get("base_users")) ldap_connection.delete("cn=ndRoleGroups," + env.vars.get("LDAP_CONFIG").get("base_users")) - # ensure boolean values are Uppercase.. + # ensure boolean values are Uppercase.. this comes from the ansible yml # (not yet implemented, probably not needed) # loop through the role files From 5977d8c908e7fb396f5ce7340a40e58ead157b79 Mon Sep 17 00:00:00 2001 From: Seb Norris Date: Tue, 22 Aug 2023 15:08:32 +0100 Subject: [PATCH 10/29] clean up home area function --- cli/ldap/update_user_home_areas.py | 61 ++++++++++-------------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py index a4d1ed8..0839119 100644 --- a/cli/ldap/update_user_home_areas.py +++ b/cli/ldap/update_user_home_areas.py @@ -2,47 +2,26 @@ from cli import config from cli.ldap import ldap_connect +import ldap3 +from ldap3 import MODIFY_REPLACE -## -# Original ansible to replicate then remove -## - # - name: Search for matching users (Home Area=Old value, Active) - # shell: > - # ldapsearch -LLL -H ldap:// -D cn=root,dc=moj,dc=com -w "${ldap_admin_password}" -b {{ user_dn }} -s one \ - # '(&(objectclass=NDUser)(userHomeArea={{ old_home_area }})(!(cn={{ old_home_area }}))(!(endDate=*)))' dn \ - # | sed 's/^$/changetype: modify\nreplace: userHomeArea\nuserHomeArea: {{ new_home_area }}\n/' \ - # > {{ workspace }}/update.ldif - # environment: - # ldap_admin_password: '{{ ldap_admin_password.stdout }}' - - # - name: Create additional LDIF for deleting home areas (required due to LDAP issue causing old values to get "stuck") - # shell: > - # cat {{ workspace }}/update.ldif \ - # | sed 's/replace: userHomeArea/delete: userHomeArea/' \ - # | sed 's/userHomeArea: {{ new_home_area }}/userHomeArea: {{ old_home_area }}/' \ - # > {{ workspace }}/delete.ldif - - # - name: Apply changes - # shell: ldapmodify -Y EXTERNAL -H ldapi:// -c -f {{ workspace }}/update.ldif 2>&1 | tee {{ workspace }}/update.log - - # - name: Restart LDAP (required due to LDAP issue causing old values to get "stuck") - # shell: systemctl restart slapd - - # - name: Force deletion of old home area (required due to LDAP issue causing old values to get "stuck") - # shell: ldapmodify -Y EXTERNAL -H ldapi:// -c -f {{ workspace }}/delete.ldif 2>&1 | tee {{ workspace }}/delete.log - - -def update_user_home_areas(old_home_area, new_home_area): +def update_user_home_areas(old_home_area, new_home_area, attribute="userHomeArea"): logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") - ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) - ldap_connection.search("ou=Users,dc=moj,dc=com", f"(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))") - - # Output records for debug purposes - records = ldap_connection.response - logging.info(len(records)) - for record in records: - logging.info(record) - - # Enter code here modify and delete LDAP entries as per the outcome of the native ansible-invoked LDAP commands above - # But using the python package methods rather than constructing LDIFs + conn = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) + + base_dn = "ou=Users,dc=moj,dc=com" # change this + search_filter = f"'(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))'" # and this + conn.search(base_dn, search_filter, attributes=[attribute]) + + # Iterate through the search results and update the attribute + for entry in conn.entries: + dn = entry.entry_dn + changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} + conn.modify(dn, changes) + + # Check if the modification was successful + if conn.result['result'] == 0: + print(f'Successfully updated {attribute} for {dn}') + else: + print(f'Failed to update {attribute} for {dn}: {conn.result}') \ No newline at end of file From 8ed92256921a0f75575bc004db2f676581ed05f5 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Tue, 22 Aug 2023 15:37:33 +0100 Subject: [PATCH 11/29] add setuptools requirements --- .gitignore | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6887167..4024106 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ # VSCode Config -.vscode \ No newline at end of file +.vscode +.secrets +.vars diff --git a/setup.py b/setup.py index 4821b95..de7e4ac 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ version="0.1", packages=find_packages(), # install_requires=["Click", "ldap3", "oracledb"], - install_requires=["Click", "ldap3"], + install_requires=["Click", "ldap3", "PyGithub", "python-dotenv", "pyjwt", "GitPython"], entry_points=""" [console_scripts] ldap-automation=cli:main_group From 8dbcf548cd44f3c1939ad8f5819c5ccfc633bdcc Mon Sep 17 00:00:00 2001 From: George Taylor Date: Tue, 22 Aug 2023 15:38:26 +0100 Subject: [PATCH 12/29] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index de7e4ac..b8e1107 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ name="ldap-automation", version="0.1", packages=find_packages(), - # install_requires=["Click", "ldap3", "oracledb"], install_requires=["Click", "ldap3", "PyGithub", "python-dotenv", "pyjwt", "GitPython"], entry_points=""" [console_scripts] From 11e13f9f25f893f7ccbc2aac81decd26f7ffdade Mon Sep 17 00:00:00 2001 From: George Taylor Date: Tue, 22 Aug 2023 15:43:31 +0100 Subject: [PATCH 13/29] remove quotes unneeded --- cli/ldap/update_user_home_areas.py | 32 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py index 0839119..a7af39a 100644 --- a/cli/ldap/update_user_home_areas.py +++ b/cli/ldap/update_user_home_areas.py @@ -7,21 +7,23 @@ def update_user_home_areas(old_home_area, new_home_area, attribute="userHomeArea"): - logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") - conn = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) + logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") + conn = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) - base_dn = "ou=Users,dc=moj,dc=com" # change this - search_filter = f"'(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))'" # and this - conn.search(base_dn, search_filter, attributes=[attribute]) + base_dn = "ou=Users,dc=moj,dc=com" # change this + search_filter = ( + f"(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" # and this + ) + conn.search(base_dn, search_filter, attributes=[attribute]) - # Iterate through the search results and update the attribute - for entry in conn.entries: - dn = entry.entry_dn - changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} - conn.modify(dn, changes) + # Iterate through the search results and update the attribute + for entry in conn.entries: + dn = entry.entry_dn + changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} + conn.modify(dn, changes) - # Check if the modification was successful - if conn.result['result'] == 0: - print(f'Successfully updated {attribute} for {dn}') - else: - print(f'Failed to update {attribute} for {dn}: {conn.result}') \ No newline at end of file + # Check if the modification was successful + if conn.result["result"] == 0: + print(f"Successfully updated {attribute} for {dn}") + else: + print(f"Failed to update {attribute} for {dn}: {conn.result}") From 84771a2bacf0d2c8ff722dd8d2fe42088a262e40 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Wed, 23 Aug 2023 13:00:22 +0100 Subject: [PATCH 14/29] Retrofit logging and env dict from rbac uplift (#17) * flexibility * logging * add shorthand options * options for log levels * Update logging.py * Update __init__.py --- cli/__init__.py | 22 +++++++++++++--------- cli/ldap/update_user_home_areas.py | 27 ++++++++++++++------------- cli/logging.py | 9 ++++++--- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/cli/__init__.py b/cli/__init__.py index 9451cc3..706b9fd 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,5 +1,5 @@ import click -import cli.ldap.add_roles_to_username, cli.ldap.rbac,cli.ldap.update_user_home_areas +import cli.ldap.add_roles_to_username, cli.ldap.rbac, cli.ldap.update_user_home_areas from cli import git import cli.env @@ -8,23 +8,27 @@ @click.group() def main_group(): pass - + + @click.command() -@click.option("--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") +@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") +@click.option("-r", "--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") @click.argument("user-role-list", required=True) def add_roles_to_users(user_ou, root_dn, user_role_list): cli.ldap.add_roles_to_username.process_user_roles_list(user_role_list, user_ou, root_dn) + # Update user home area @click.command() -@click.option("--old-home-area", help="name of old home area") -@click.option("--new-home-area", help="name of new home area") +@click.option("-o", "--old-home-area", help="name of old home area", required=True) +@click.option("-n", "--new-home-area", help="name of new home area", required=True) def update_user_home_areas(old_home_area, new_home_area): - cli.ldap.update_user_home_areas.update_user_home_areas(old_home_area, new_home_area) - + base_dn = env.vars.get("LDAP_CONFIG").get("base_users") + cli.ldap.update_user_home_areas.update_user_home_areas(old_home_area, new_home_area, base_dn) + + @click.command() -@click.option("--rbac-repo-tag", help="RBAC repo tag to use", default="master") +@click.option("-t", "--rbac-repo-tag", help="RBAC repo tag to use", default="master") def rbac_uplift(rbac_repo_tag): cli.ldap.rbac.main(rbac_repo_tag) diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py index a7af39a..1a34b3f 100644 --- a/cli/ldap/update_user_home_areas.py +++ b/cli/ldap/update_user_home_areas.py @@ -1,29 +1,30 @@ -import logging +from cli.logging import log +from cli import env -from cli import config from cli.ldap import ldap_connect import ldap3 from ldap3 import MODIFY_REPLACE -def update_user_home_areas(old_home_area, new_home_area, attribute="userHomeArea"): - logging.info(f"Updating user home areas from {old_home_area} to {new_home_area}") - conn = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) +def update_user_home_areas(old_home_area, new_home_area, base_dn, attribute="userHomeArea", object_class="NDUser"): + log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) - base_dn = "ou=Users,dc=moj,dc=com" # change this search_filter = ( - f"(&(objectclass=NDUser)(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" # and this + f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" ) - conn.search(base_dn, search_filter, attributes=[attribute]) + ldap_connection.search(base_dn, search_filter, attributes=[attribute]) # Iterate through the search results and update the attribute - for entry in conn.entries: + for entry in ldap_connection.entries: dn = entry.entry_dn changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} - conn.modify(dn, changes) + ldap_connection.modify(dn, changes) # Check if the modification was successful - if conn.result["result"] == 0: - print(f"Successfully updated {attribute} for {dn}") + if ldap_connection.result["result"] == 0: + log.info(f"Successfully updated {attribute} for {dn}") else: - print(f"Failed to update {attribute} for {dn}: {conn.result}") + log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") diff --git a/cli/logging.py b/cli/logging.py index 0b78a11..4ab4238 100644 --- a/cli/logging.py +++ b/cli/logging.py @@ -1,5 +1,5 @@ import logging -from cli.env import secrets +import cli.env class SensitiveFormatter(logging.Formatter): @@ -7,7 +7,7 @@ class SensitiveFormatter(logging.Formatter): @staticmethod def _filter(s): - secrets_set = set(secrets.values()) + secrets_set = set(cli.env.secrets.values()) redacted = " ".join([s.replace(secret, "*" * len(secret)) for secret in secrets_set if secret is not None]) return redacted @@ -16,7 +16,10 @@ def format(self, record): return self._filter(original) -logging.basicConfig(level=logging.DEBUG) +if cli.env.vars.get("DEBUG") == 1: + logging.basicConfig(level=logging.DEBUG) +else: + logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) for handler in log.root.handlers: handler.setFormatter(SensitiveFormatter()) From 5efaeb07daac00012eff7d26c17f84764644af03 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Wed, 6 Sep 2023 11:03:07 +0100 Subject: [PATCH 15/29] Nit 824 nit 823 - update user roles and user notes (#18) * new functions and structure * find common entries in both * refactor + python rewrite foruser roles * remove action * remove debugging * start oracle db * add update notes * typo + rm commented code * refactor + comments * Update __init__.py * fix logger duplicates * re format + remove print debugging * log levels + debugging * Update logger.py * fixes requirements * reformat connection for oracle * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * bind by name * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * add handling for user notes --- cli/__init__.py | 48 +++++- cli/database/__init__.py | 13 ++ cli/git/__init__.py | 2 - cli/ldap/__init__.py | 4 +- cli/ldap/add_roles_to_username.py | 42 ------ cli/ldap/rbac.py | 90 +++++++----- cli/ldap/update_user_home_areas.py | 30 ---- cli/ldap/user.py | 229 +++++++++++++++++++++++++++++ cli/logger.py | 53 +++++++ cli/logging.py | 25 ---- setup.py | 11 +- 11 files changed, 405 insertions(+), 142 deletions(-) create mode 100644 cli/database/__init__.py delete mode 100644 cli/ldap/add_roles_to_username.py delete mode 100644 cli/ldap/update_user_home_areas.py create mode 100644 cli/ldap/user.py create mode 100644 cli/logger.py delete mode 100644 cli/logging.py diff --git a/cli/__init__.py b/cli/__init__.py index 706b9fd..fab1241 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,8 +1,7 @@ import click -import cli.ldap.add_roles_to_username, cli.ldap.rbac, cli.ldap.update_user_home_areas +import cli.ldap.rbac, cli.ldap.user -from cli import git -import cli.env +from cli import git, logger @click.group() @@ -15,16 +14,48 @@ def main_group(): @click.option("-r", "--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") @click.argument("user-role-list", required=True) def add_roles_to_users(user_ou, root_dn, user_role_list): - cli.ldap.add_roles_to_username.process_user_roles_list(user_role_list, user_ou, root_dn) + cli.ldap.user.process_user_roles_list(user_role_list, user_ou, root_dn) # Update user home area @click.command() @click.option("-o", "--old-home-area", help="name of old home area", required=True) @click.option("-n", "--new-home-area", help="name of new home area", required=True) -def update_user_home_areas(old_home_area, new_home_area): - base_dn = env.vars.get("LDAP_CONFIG").get("base_users") - cli.ldap.update_user_home_areas.update_user_home_areas(old_home_area, new_home_area, base_dn) +@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") +@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") +def update_user_home_areas(old_home_area, new_home_area, user_ou, root_dn): + cli.ldap.user.change_home_areas(old_home_area, new_home_area, user_ou, root_dn) + + +# Update user roles +@click.command() +@click.argument("roles", required=True) +@click.argument("user-note", required=False) +@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") +@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") +@click.option("--add", help="Add role to users", is_flag=True) +@click.option("--remove", help="Remove role from users", is_flag=True) +@click.option("--update-notes", help="Remove role from users", is_flag=True) +@click.option( + "-rf", + "--role-filter", + help='Comma seperated string to generate roles filter from eg "role1,role2,role3"', + required=False, + default="*", +) +@click.option("-uf", "--user-filter", help="Filter to find users", required=False, default="(userSector=*)") +def update_user_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, user_filter, role_filter): + cli.ldap.user.update_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note=user_note, + user_filter=user_filter, + role_filter=role_filter, + ) @click.command() @@ -38,6 +69,9 @@ def rbac_uplift(rbac_repo_tag): main_group.add_command(add_roles_to_users) main_group.add_command(rbac_uplift) main_group.add_command(update_user_home_areas) +main_group.add_command(update_user_roles) + +logger.configure_logging() if __name__ == "__main__": main_group() diff --git a/cli/database/__init__.py b/cli/database/__init__.py new file mode 100644 index 0000000..9b64657 --- /dev/null +++ b/cli/database/__init__.py @@ -0,0 +1,13 @@ +import oracledb +from cli import env +from cli.logger import log + + +def connection(): + try: + conn = oracledb.connect(env.secrets.get("DB_CONNECTION_STRING")) + log.debug("Created database connection successfully") + return conn + except Exception as e: + log.exception(e) + raise e diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 5378e65..2486329 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -1,10 +1,8 @@ -from github import Github, Auth from git import Repo import jwt import time import requests import logging -from cli import env def get_access_token(app_id, private_key, installation_id): diff --git a/cli/ldap/__init__.py b/cli/ldap/__init__.py index a969c09..07fe481 100644 --- a/cli/ldap/__init__.py +++ b/cli/ldap/__init__.py @@ -1,10 +1,10 @@ from ldap3 import Server, Connection, ALL -from logging import log # import oracledb def ldap_connect(ldap_host, ldap_user, ldap_password): - server = Server(ldap_host, get_info=ALL) + server = Server(ldap_host) + return Connection( server=server, user=ldap_user, password=ldap_password, auto_bind="NO_TLS", authentication="SIMPLE" ) diff --git a/cli/ldap/add_roles_to_username.py b/cli/ldap/add_roles_to_username.py deleted file mode 100644 index c1736a9..0000000 --- a/cli/ldap/add_roles_to_username.py +++ /dev/null @@ -1,42 +0,0 @@ -from cli.logging import log -from cli import env -from cli.ldap import ldap_connect - - -def parse_user_role_list(user_role_list): - # The format of the list should be a pipe separated list of username and role lists, - # where the username and role list is separated by a comma character, - # and the roles are separated by a semi-colon: - # username1,role1;role2;role3|username2,role1;role2 - - return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} - - -def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - log.info(f"Adding roles {roles} to user {username}") - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - for role in roles: - ldap_connection.add( - f"cn={role},cn={username},{user_ou},{root_dn}", - attributes={ - "objectClass": ["NDRoleAssociation", "alias"], - "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{root_dn}", - }, - ) - if ldap_connection.result["result"] == 0: - print(f"Successfully added role {role} to user {username}") - elif ldap_connection.result["result"] == 68: - print(f"Role {role} already exists for user {username}") - else: - print(ldap_connection.result) - print(ldap_connection.response) - raise Exception(f"Failed to add role {role} to user {username}") - - -def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - log.info(f"secrets: {env.secrets}") - user_roles = parse_user_role_list(user_role_list) - for user, roles in user_roles.items(): - add_roles_to_user(user, roles, user_ou, root_dn) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index c942a1f..a73adae 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -1,11 +1,9 @@ -import re - import ldap3.utils.hashed from cli.ldap import ldap_connect from cli import env import cli.git as git import glob -from cli.logging import log +from cli.logger import log from pathlib import Path import cli.template from ldif import LDIFParser @@ -73,10 +71,15 @@ def context_ldif(rendered_files): env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") ) ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add context {dn}, status: {ldap_connection.result['result']}") + print(ldap_connection.result["result"]) + if ldap_connection.result["result"] == 0: + print(f"Successfully added context") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") def group_ldifs(rendered_files): @@ -98,12 +101,14 @@ def group_ldifs(rendered_files): if record.get("description"): print("updating description") ldap_connection.modify(dn, {"description": [(ldap3.MODIFY_REPLACE, record["description"])]}) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception( - f"Failed to update description for group {dn}, status: {ldap_connection.result['result']}" - ) + if ldap_connection.result["result"] == 0: + print(f"Successfully added groups") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") def policy_ldifs(rendered_files): @@ -126,10 +131,14 @@ def policy_ldifs(rendered_files): # print(record) # add the record to ldap ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add policy {dn}, status: {ldap_connection.result['result']}") + if ldap_connection.result["result"] == 0: + print(f"Successfully added policies") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") def role_ldifs(rendered_files): @@ -156,10 +165,14 @@ def role_ldifs(rendered_files): # print(record) # add the record to ldap ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add role {dn}, status: {ldap_connection.result['result']}") + if ldap_connection.result["result"] == 0: + print(f"Successfully added roles") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") # not complete!! @@ -182,10 +195,14 @@ def schema_ldifs(rendered_files): # print(record) # add the record to ldap ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add schema {dn}, status: {ldap_connection.result['result']}") + if ldap_connection.result["result"] == 0: + print(f"Successfully added schemas") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") def user_ldifs(rendered_files): @@ -202,13 +219,16 @@ def user_ldifs(rendered_files): # loop through the records for dn, record in parser.parse(): print("got entry record: %s" % dn) + + # for each user find child entries and delete them + ldap_connection.search(dn, "(objectclass=*)", search_scope=ldap3.SUBTREE) + for entry in ldap_connection.entries: + print(entry.entry_dn) + ldap_connection.delete(entry.entry_dn) + # print(record) # add the record to ldap ldap_connection.delete(dn) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to delete user {dn}, status: {ldap_connection.result['result']}") # loop through the user files for file in user_files: @@ -220,10 +240,14 @@ def user_ldifs(rendered_files): # print(record) # add the record to ldap ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add user {dn}, status: {ldap_connection.result['result']}") + if ldap_connection.result["result"] == 0: + print(f"Successfully added users") + elif ldap_connection.result["result"] == 68: + print(f"{dn} already exists") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add {dn}... {record}") def main(rbac_repo_tag, clone_path="./rbac"): diff --git a/cli/ldap/update_user_home_areas.py b/cli/ldap/update_user_home_areas.py deleted file mode 100644 index 1a34b3f..0000000 --- a/cli/ldap/update_user_home_areas.py +++ /dev/null @@ -1,30 +0,0 @@ -from cli.logging import log -from cli import env - -from cli.ldap import ldap_connect -import ldap3 -from ldap3 import MODIFY_REPLACE - - -def update_user_home_areas(old_home_area, new_home_area, base_dn, attribute="userHomeArea", object_class="NDUser"): - log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - - search_filter = ( - f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" - ) - ldap_connection.search(base_dn, search_filter, attributes=[attribute]) - - # Iterate through the search results and update the attribute - for entry in ldap_connection.entries: - dn = entry.entry_dn - changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} - ldap_connection.modify(dn, changes) - - # Check if the modification was successful - if ldap_connection.result["result"] == 0: - log.info(f"Successfully updated {attribute} for {dn}") - else: - log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") diff --git a/cli/ldap/user.py b/cli/ldap/user.py new file mode 100644 index 0000000..16b6fed --- /dev/null +++ b/cli/ldap/user.py @@ -0,0 +1,229 @@ +import oracledb + +import cli.ldap + +from cli.logger import log +from cli import env + +from cli.ldap import ldap_connect +from ldap3 import MODIFY_REPLACE, DEREF_NEVER + +import cli.database +from itertools import product + + +######################################### +# Change a users home area +######################################### +def change_home_areas(old_home_area, new_home_area, user_ou, root_dn, attribute="userHomeArea", object_class="NDUser"): + log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + + search_filter = ( + f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + ) + ldap_connection.search(",".join([user_ou, root_dn]), search_filter, attributes=[attribute]) + + # Iterate through the search results and update the attribute + for entry in ldap_connection.entries: + dn = entry.entry_dn + changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} + ldap_connection.modify(dn, changes) + + # Check if the modification was successful + if ldap_connection.result["result"] == 0: + log.info(f"Successfully updated {attribute} for {dn}") + else: + log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") + + +######################################### +# Add roles to a user +######################################### + + +def parse_user_role_list(user_role_list): + # The format of the list should be a pipe separated list of username and role lists, + # where the username and role list is separated by a comma character, + # and the roles are separated by a semi-colon: + # username1,role1;role2;role3|username2,role1;role2 + + return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} + + +def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): + log.info(f"Adding roles {roles} to user {username}") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + for role in roles: + ldap_connection.add( + f"cn={role},cn={username},{user_ou},{root_dn}", + attributes={ + "objectClass": ["NDRoleAssociation", "alias"], + "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", + }, + ) + if ldap_connection.result["result"] == 0: + print(f"Successfully added role {role} to user {username}") + elif ldap_connection.result["result"] == 68: + print(f"Role {role} already exists for user {username}") + else: + print(ldap_connection.result) + print(ldap_connection.response) + raise Exception(f"Failed to add role {role} to user {username}") + + +def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): + log.info(f"secrets: {env.secrets}") + user_roles = parse_user_role_list(user_role_list) + for user, roles in user_roles.items(): + add_roles_to_user(user, roles, user_ou, root_dn) + + +######################################### +# Update user roles +######################################### + + +def update_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter="(userSector=*)", + role_filter="*", +): + if update_notes and (user_note is None or len(user_note) < 1): + log.error("User note must be provided when updating notes") + raise Exception("User note must be provided when updating notes") + ldap_connection_user_filter = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + + # # Search for users matching the user_filter + ldap_connection_user_filter.search(",".join([user_ou, root_dn]), user_filter, attributes=["cn"]) + users_found = sorted([entry.cn.value for entry in ldap_connection_user_filter.entries if entry.cn.value]) + log.debug("users found from user filter") + log.debug(users_found) + ldap_connection_user_filter.unbind() + + roles_filter_list = role_filter.split(",") + roles = roles.split(",") + + # create role filter + if len(roles_filter_list) > 0: + full_role_filter = ( + f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" + ) + else: + full_role_filter = "(&(objectclass=NDRoleAssociation)(cn=*))" + + # Search for roles matching the role_filter + ldap_connection_role_filter = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + + ldap_connection_role_filter.search( + ",".join([user_ou, root_dn]), + full_role_filter, + attributes=["cn"], + dereference_aliases=DEREF_NEVER, + ) + roles_found = sorted( + list(set([entry.entry_dn.split(",")[1].split("=")[1] for entry in ldap_connection_role_filter.entries])) + ) + log.debug("users found from roles filter: ") + log.debug(roles_found) + + ldap_connection_role_filter.unbind() + + # generate a list of matches in roles and users + matched_users = set(users_found) & set(roles_found) + log.debug("matched users: ") + log.debug(matched_users) + + # cartesian_product = [(user, role) for user in matched_users for role in roles] + + cartesian_product = list(product(matched_users, roles)) + log.debug("cartesian product: ") + log.debug(cartesian_product) + + ldap_connection_action = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + + for item in cartesian_product: + if add: + ldap_connection_action.add( + f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}", + attributes={ + "cn": item[1], + "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", + "objectClass": ["NDRoleAssociation", "alias", "top"], + }, + ) + if ldap_connection_action.result["result"] == 0: + log.info(f"Successfully added role '{item[1]}' to user '{item[0]}'") + elif ldap_connection_action.result["result"] == 68: + log.info(f"Role '{item[1]}' already present for user '{item[0]}'") + else: + log.error(f"Failed to add role '{item[1]}' to user '{item[0]}'") + log.debug(ldap_connection_action.result) + elif remove: + ldap_connection_action.delete(f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}") + if ldap_connection_action.result["result"] == 0: + log.info(f"Successfully removed role '{item[1]}' from user '{item[0]}'") + elif ldap_connection_action.result["result"] == 32: + log.info(f"Role '{item[1]}' already absent for user '{item[0]}'") + else: + log.error(f"Failed to remove role '{item[1]}' from user '{item[0]}'") + log.debug(ldap_connection_action.result) + else: + log.error("No action specified") + + if update_notes: + connection = cli.database.connection() + log.debug("Created database cursor successfully") + for user in matched_users: + try: + update_sql = "UPDATE USER_ SET LAST_UPDATED_DATETIME=CURRENT_DATE, LAST_UPDATED_USER_ID=4 WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + update_cursor = connection.cursor() + update_cursor.execute(update_sql, [user]) + update_cursor.close() + + insert_sql = """ + INSERT INTO USER_NOTE ( + USER_NOTE_ID, + USER_ID, + LAST_UPDATED_USER_ID, + LAST_UPDATED_DATETIME, + NOTES + ) + SELECT + user_note_id_seq.nextval, + USER_ID, + 4, + sysdate, + :user_note + FROM + USER_ + WHERE + UPPER(DISTINGUISHED_NAME) = UPPER(:user_dn) + """ + insert_cursor = connection.cursor() + insert_cursor.setinputsizes(user_note=oracledb.CLOB) + insert_cursor.execute(insert_sql, user_note=user_note, user_dn=user) + insert_cursor.close() + + log.info(f"Updated notes for user {user}") + connection.commit() + log.info("Committed changes to database successfully") + except: + log.exception(f"Failed to update notes for user {user}") + connection.close() diff --git a/cli/logger.py b/cli/logger.py new file mode 100644 index 0000000..aba6e5f --- /dev/null +++ b/cli/logger.py @@ -0,0 +1,53 @@ +import logging +import cli.env + + +def configure_logging(): + class SensitiveFormatter(logging.Formatter): + """Formatter that removes secrets from log messages.""" + + def __init__(self, format_str=None, datefmt_str=None): + super().__init__(fmt=format_str, datefmt=datefmt_str) + self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here + self.default_msec_format = "%s.%03d" + + def _filter(self, s): + redacted = " ".join( + ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] + ) + + return redacted + + def format(self, record): + original = super().format(record) + return self._filter(original) + + print("configure_logging") + """Configure logging based on environment variables.""" + format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" + + log = logging.getLogger(__name__) + + if logging.root.hasHandlers(): + logging.root.handlers = [] + + handler = logging.StreamHandler() + handler.setFormatter(SensitiveFormatter(format_str=format, datefmt_str=datefmt)) + logging.root.addHandler(handler) + if cli.env.vars.get("LOG_LEVEL") == "DEBUG": + print("DEBUG") + log.setLevel(logging.DEBUG) + elif cli.env.vars.get("LOG_LEVEL") == "WARNING": + print("WARNING") + log.setLevel(logging.WARNING) + elif cli.env.vars.get("LOG_LEVEL") == "ERROR": + print("ERROR") + log.setLevel(logging.ERROR) + elif cli.env.vars.get("LOG_LEVEL") == "INFO": + print("INFO") + log.setLevel(logging.INFO) + return True + + +log = logging.getLogger(__name__) diff --git a/cli/logging.py b/cli/logging.py deleted file mode 100644 index 4ab4238..0000000 --- a/cli/logging.py +++ /dev/null @@ -1,25 +0,0 @@ -import logging -import cli.env - - -class SensitiveFormatter(logging.Formatter): - """Formatter that removes secrets from log messages.""" - - @staticmethod - def _filter(s): - secrets_set = set(cli.env.secrets.values()) - redacted = " ".join([s.replace(secret, "*" * len(secret)) for secret in secrets_set if secret is not None]) - return redacted - - def format(self, record): - original = logging.Formatter.format(self, record) - return self._filter(original) - - -if cli.env.vars.get("DEBUG") == 1: - logging.basicConfig(level=logging.DEBUG) -else: - logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) -for handler in log.root.handlers: - handler.setFormatter(SensitiveFormatter()) diff --git a/setup.py b/setup.py index b8e1107..cd85c83 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,19 @@ from setuptools import setup, find_packages +# Read requirements from requirements.txt +with open("requirements.txt") as f: + requirements = f.read().splitlines() + +standard_pkgs = [r for r in requirements if not r.startswith("git+")] +git_pkgs = [r for r in requirements if r.startswith("git+")] +formatted_git_pkgs = [f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs] +all_reqs = standard_pkgs + formatted_git_pkgs + setup( name="ldap-automation", version="0.1", packages=find_packages(), - install_requires=["Click", "ldap3", "PyGithub", "python-dotenv", "pyjwt", "GitPython"], + install_requires=all_reqs, entry_points=""" [console_scripts] ldap-automation=cli:main_group From e5b80255106ded79949a59568717424697bbb11b Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 8 Sep 2023 15:06:47 +0100 Subject: [PATCH 16/29] Nit 822 (#19) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt --------- Co-authored-by: Seb Norris --- cli/__init__.py | 8 +++++ cli/ldap/user.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/cli/__init__.py b/cli/__init__.py index fab1241..8c42fa8 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -64,12 +64,20 @@ def rbac_uplift(rbac_repo_tag): cli.ldap.rbac.main(rbac_repo_tag) +@click.command() +@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") +@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") +def deactivate_crc_users(user_ou, root_dn): + cli.ldap.user.deactivate_crc_users(user_ou, root_dn) + + # from cli.ldap import test main_group.add_command(add_roles_to_users) main_group.add_command(rbac_uplift) main_group.add_command(update_user_home_areas) main_group.add_command(update_user_roles) +main_group.add_command(deactivate_crc_users) logger.configure_logging() diff --git a/cli/ldap/user.py b/cli/ldap/user.py index 16b6fed..3fb24ea 100644 --- a/cli/ldap/user.py +++ b/cli/ldap/user.py @@ -11,6 +11,8 @@ import cli.database from itertools import product +from datetime import datetime + ######################################### # Change a users home area @@ -227,3 +229,83 @@ def update_roles( except: log.exception(f"Failed to update notes for user {user}") connection.close() + + +######################################### +# Deactivate CRC User Accounts +######################################### + + +def deactivate_crc_users(user_ou, root_dn): + log.info("Deactivating CRC users") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) + + user_filter = "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + + home_areas = [ + [ + "C01", + "C02", + "C03", + "C04", + "C05", + "C06", + "C07", + "C08", + "C09", + "C10", + "C11", + "C12", + "C13", + "C14", + "C15", + "C16", + "C17", + "C18", + "C19", + "C20", + "C21", + ] + ] + + found_users = [] + for home_area in home_areas: + ldap_connection.search( + ",".join([user_ou, root_dn]), + f"(&(userHomeArea={home_area})(!(cn={home_area})){user_filter})", + attributes=["dn"], + ) + + found_users.append(entry.entry_dn for entry in ldap_connection.entries) + + ldap_connection.search( + ",".join([user_ou, root_dn]), + f"(&(!(userHomeArea=*)){user_filter})", + attributes=["dn"], + ) + found_users_no_home_area = [entry.entry_dn for entry in ldap_connection.entries] + + all_users = found_users + found_users_no_home_area + + date_str = f"{datetime.now().strftime('%Y%m%d')}000000Z" + + for user in all_users: + ldap_connection.modify(user, {"endDate": [(MODIFY_REPLACE, [date_str])]}) + + connection = cli.database.connection() + for user_dn in all_users: + try: + update_sql = ( + f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + ) + update_cursor = connection.cursor() + update_cursor.execute(update_sql, [user_dn]) + update_cursor.close() + log.info(f"Updated END_DATE for user {user_dn}") + connection.commit() + log.info("Committed changes to database successfully") + except: + log.exception(f"Failed to update END_DATE for user {user_dn}") + connection.close() diff --git a/requirements.txt b/requirements.txt index 4149531..3888448 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ click==8.1.6 ldap3 -oracledb==1.2.2 +oracledb==1.4 ansible-runner PyGithub GitPython From f8e0fb4ab6b7128b56883e1893433e6fea8f6b6d Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 11 Sep 2023 15:40:31 +0100 Subject: [PATCH 17/29] Nit 822 (#20) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py --------- Co-authored-by: Seb Norris --- cli/ldap/rbac.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index a73adae..0a0125a 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -25,8 +25,24 @@ def get_repo(repo_tag="master"): def prep_for_templating(files, strings=None): + rbac_substitutions = { + "bind_password_hash.stdout": "bind_password_hash", + r"ldap_config.base_users | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_users_ou", + r"ldap_config.base_root | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_root_dc", + r"ldap_config.base_groups | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_groups_ou", + r"ldap_config.bind_user | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.bind_user_cn", + "'/'+environment_name+'/'+project_name+'": "", + "/gdpr/api/": "'gdpr_api_", + "/pwm/pwm/config_password": "'pwm_config_password", + "/merge/api/client_secret": "'merge_api_client_secret", + "/weblogic/ndelius-domain/umt_client_secret": "'umt_client_secret", + "ssm_prefix + ": "", + "cn=Users,dc=pcms,dc=internal": "ou=Users,dc=moj,dc=com", + "ssm_prefix+": "", + } + if strings is None: - strings = env.vars.get("RBAC_SUBSTITUTIONS") + strings = env.vars.get("RBAC_SUBSTITUTIONS") or rbac_substitutions # get a list of files # print(strings) # print(type(strings)) @@ -45,10 +61,25 @@ def prep_for_templating(files, strings=None): def template_rbac(files): hashed_pwd_admin_user = ldap3.utils.hashed.hashed(ldap3.HASHED_SALTED_SHA, env.secrets.get("LDAP_ADMIN_PASSWORD")) rendered_files = [] + + ldap_config = { + "bind_user": "cn=root,dc=moj,dc=com", + "bind_user_cn": "root", + "base_root": "dc=moj,dc=com", + "base_root_dc": "moj", + "base_users": "ou=Users,dc=moj,dc=com", + "base_users_ou": "Users", + "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", + "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", + "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", + "base_groups": "ou=groups,dc=moj,dc=com", + "base_groups_ou": "groups", + } + for file in files: rendered_text = cli.template.render( file, - ldap_config=env.vars.get("LDAP_CONFIG"), + ldap_config=env.vars.get("LDAP_CONFIG") or ldap_config, bind_password_hash=hashed_pwd_admin_user, secrets=env.secrets, oasys_password=env.secrets.get("OASYS_PASSWORD"), @@ -252,8 +283,6 @@ def user_ldifs(rendered_files): def main(rbac_repo_tag, clone_path="./rbac"): repo = get_repo(rbac_repo_tag) - print(env.vars.get("RBAC_SUBSTITUTIONS")) - files = [ file for file in glob.glob(f"{clone_path}/**/*", recursive=True) From 7c1a1ccf8c685c96fe7b1203c31adedf4389c066 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 11 Sep 2023 16:26:52 +0100 Subject: [PATCH 18/29] Nit 822 (#21) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac --------- Co-authored-by: Seb Norris --- cli/ldap/rbac.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 0a0125a..e17189e 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -8,16 +8,18 @@ import cli.template from ldif import LDIFParser +#### example for token auth +# def get_repo_with_token(repo_tag="master"): +# app_id = env.vars.get("GH_APP_ID") +# private_key = env.vars.get("GH_PRIVATE_KEY") +# installation_id = env.vars.get("GH_INSTALLATION_ID") +# token = git.get_access_token(app_id, private_key, installation_id) + def get_repo(repo_tag="master"): - app_id = env.vars.get("GH_APP_ID") - private_key = env.vars.get("GH_PRIVATE_KEY") - installation_id = env.vars.get("GH_INSTALLATION_ID") - # url = 'https://github.com/ministryofjustice/hmpps-delius-pipelines.git' url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" - token = git.get_access_token(app_id, private_key, installation_id) try: - repo = git.get_repo(url, token=token, dest_name="rbac", branch_or_tag=repo_tag) + repo = git.get_repo(url, dest_name="rbac", branch_or_tag=repo_tag) return repo except Exception as e: log.exception(e) From cf13288d3fb411d1ecc396bebc17abdc88305e8a Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 11 Sep 2023 16:49:21 +0100 Subject: [PATCH 19/29] Nit 822 (#22) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac * Update rbac.py --------- Co-authored-by: Seb Norris --- cli/ldap/rbac.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index e17189e..2662a65 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -29,10 +29,10 @@ def get_repo(repo_tag="master"): def prep_for_templating(files, strings=None): rbac_substitutions = { "bind_password_hash.stdout": "bind_password_hash", - r"ldap_config.base_users | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_users_ou", - r"ldap_config.base_root | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_root_dc", - r"ldap_config.base_groups | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.base_groups_ou", - r"ldap_config.bind_user | regex_replace(\'^.+?=(.+?),.*$\', \'\\\1\')": "ldap_config.bind_user_cn", + r"ldap_config.base_users | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_users_ou", + r"ldap_config.base_root | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_root_dc", + r"ldap_config.base_groups | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_groups_ou", + r"ldap_config.bind_user | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.bind_user_cn", "'/'+environment_name+'/'+project_name+'": "", "/gdpr/api/": "'gdpr_api_", "/pwm/pwm/config_password": "'pwm_config_password", @@ -52,9 +52,6 @@ def prep_for_templating(files, strings=None): file = Path(file_path) for k, v in strings.items(): print("replacing", k, "with", v, "in", file_path) - # print( - # file.read_text().replace(k, v), - # ) file.write_text( file.read_text().replace(k, v), ) From abd4171aefe457a5585579702d75a3f403d8b236 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 11 Sep 2023 17:06:11 +0100 Subject: [PATCH 20/29] Nit 822 (#23) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac * Update rbac.py * ldap config dict or local val --------- Co-authored-by: Seb Norris --- cli/ldap/rbac.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 2662a65..240b090 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -16,6 +16,21 @@ # token = git.get_access_token(app_id, private_key, installation_id) +ldap_config = { + "bind_user": "cn=root,dc=moj,dc=com", + "bind_user_cn": "root", + "base_root": "dc=moj,dc=com", + "base_root_dc": "moj", + "base_users": "ou=Users,dc=moj,dc=com", + "base_users_ou": "Users", + "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", + "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", + "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", + "base_groups": "ou=groups,dc=moj,dc=com", + "base_groups_ou": "groups", +} + + def get_repo(repo_tag="master"): url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" try: @@ -61,20 +76,6 @@ def template_rbac(files): hashed_pwd_admin_user = ldap3.utils.hashed.hashed(ldap3.HASHED_SALTED_SHA, env.secrets.get("LDAP_ADMIN_PASSWORD")) rendered_files = [] - ldap_config = { - "bind_user": "cn=root,dc=moj,dc=com", - "bind_user_cn": "root", - "base_root": "dc=moj,dc=com", - "base_root_dc": "moj", - "base_users": "ou=Users,dc=moj,dc=com", - "base_users_ou": "Users", - "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", - "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", - "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", - "base_groups": "ou=groups,dc=moj,dc=com", - "base_groups_ou": "groups", - } - for file in files: rendered_text = cli.template.render( file, @@ -149,7 +150,8 @@ def policy_ldifs(rendered_files): policy_files = [file for file in rendered_files if "policy" in Path(file).name] # first, delete the policies - ldap_connection.delete("ou=Policies," + env.vars.get("LDAP_CONFIG").get("base_root")) + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + ldap_connection.delete("ou=Policies," + ldap_config_dict.get("base_root")) # loop through the policy files for file in policy_files: @@ -179,8 +181,9 @@ def role_ldifs(rendered_files): role_files = [file for file in rendered_files if "nd_role" in Path(file).name] # first, delete the roles - ldap_connection.delete("cn=ndRoleCatalogue," + env.vars.get("LDAP_CONFIG").get("base_users")) - ldap_connection.delete("cn=ndRoleGroups," + env.vars.get("LDAP_CONFIG").get("base_users")) + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + ldap_connection.delete("cn=ndRoleCatalogue," + ldap_config_dict.get("base_users")) + ldap_connection.delete("cn=ndRoleGroups," + ldap_config_dict.get("base_users")) # ensure boolean values are Uppercase.. this comes from the ansible yml # (not yet implemented, probably not needed) From 8e08058a0be35b56783118fc9dfc6a041c2a987d Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 22 Sep 2023 16:26:47 +0100 Subject: [PATCH 21/29] Formatting & linting pre commits (#24) * add pre commit * Update readme.md * format * Update tag-and-release.yml * Update pyproject.toml * Update .flake8 * Update .flake8 * use black defualt * format to black defaults * update black to latest * remove boilerplate excludes --- .flake8 | 34 ++++ .github/workflows/tag-and-release.yml | 2 +- .pre-commit-config.yaml | 43 ++++ cli/__init__.py | 179 ++++++++++++++--- cli/database/__init__.py | 8 +- cli/env.py | 50 ++++- cli/git/__init__.py | 62 +++++- cli/ldap/__init__.py | 17 +- cli/ldap/rbac.py | 276 +++++++++++++++++++++----- cli/ldap/user.py | 244 ++++++++++++++++++----- cli/logger.py | 42 +++- cli/template/__init__.py | 43 +++- pyproject.toml | 16 ++ readme.md | 18 +- requirements-dev.txt | 14 ++ setup.py | 9 +- 16 files changed, 883 insertions(+), 174 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7ca5edc --- /dev/null +++ b/.flake8 @@ -0,0 +1,34 @@ +[flake8] + +# PEP-8 line length is not very practical +max-line-length = 88 + +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + # flake8/pycodechecker give false positives on black code + # line break before ':' apparently gives false positives with black formatter... + E203, + # line break before binary operator, fights with black formatter... + W503, + # importing with '*' ... + F403, + # Bare exception handling, fixing in NIT-854 + E722, + # Missing docstring in public nested class + D104, + # Missing docstring in public package + D103, + # f-string but no variables, e.g. print(f"hello")... + F541, + # Line too long (>79 chars), but should not be firing due to max-line-length = 120 + E501, + # add docustrings + D100, + # to be corrected with NIT-854 + B001 + +# ===================== +# flake-quote settings: +# ===================== +# Set this to match black style: +inline-quotes = double \ No newline at end of file diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index f1d1b2a..6312147 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -56,4 +56,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create ${{ steps.tag.outputs.new_tag }} --title "Release ${{ steps.tag.outputs.new_tag }}" --generate-notes --verify-tag \ No newline at end of file + gh release create ${{ steps.tag.outputs.new_tag }} --title "Release ${{ steps.tag.outputs.new_tag }}" --generate-notes --verify-tag diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1e1b806 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# pre-commit run --all-files +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-yaml + - id: debug-statements + exclude: tests/ + - id: destroyed-symlinks + - id: end-of-file-fixer + exclude: tests/test_changes/ + files: \.(py|sh|rst|yml|yaml)$ + - id: mixed-line-ending + - id: trailing-whitespace + files: \.(py|sh|rst|yml|yaml)$ + - repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: [ + 'flake8-blind-except', + 'flake8-docstrings', + 'flake8-bugbear', + 'flake8-comprehensions', + 'flake8-docstrings', + 'flake8-implicit-str-concat', + 'pydocstyle>=5.0.0', + ] + - repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + files: \.(py|sh|rst|yml|yaml)$ diff --git a/cli/__init__.py b/cli/__init__.py index 8c42fa8..4ea2b0b 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,7 +1,10 @@ import click -import cli.ldap.rbac, cli.ldap.user +import cli.ldap.rbac +import cli.ldap.user -from cli import git, logger +from cli import ( + logger, +) @click.group() @@ -10,41 +13,136 @@ def main_group(): @click.command() -@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("-r", "--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") -@click.argument("user-role-list", required=True) -def add_roles_to_users(user_ou, root_dn, user_role_list): - cli.ldap.user.process_user_roles_list(user_role_list, user_ou, root_dn) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to", + default="dc=moj,dc=com", +) +@click.argument( + "user-role-list", + required=True, +) +def add_roles_to_users( + user_ou, + root_dn, + user_role_list, +): + cli.ldap.user.process_user_roles_list( + user_role_list, + user_ou, + root_dn, + ) # Update user home area @click.command() -@click.option("-o", "--old-home-area", help="name of old home area", required=True) -@click.option("-n", "--new-home-area", help="name of new home area", required=True) -@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") -def update_user_home_areas(old_home_area, new_home_area, user_ou, root_dn): - cli.ldap.user.change_home_areas(old_home_area, new_home_area, user_ou, root_dn) +@click.option( + "-o", + "--old-home-area", + help="name of old home area", + required=True, +) +@click.option( + "-n", + "--new-home-area", + help="name of new home area", + required=True, +) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def update_user_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, +): + cli.ldap.user.change_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, + ) # Update user roles @click.command() -@click.argument("roles", required=True) -@click.argument("user-note", required=False) -@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") -@click.option("--add", help="Add role to users", is_flag=True) -@click.option("--remove", help="Remove role from users", is_flag=True) -@click.option("--update-notes", help="Remove role from users", is_flag=True) +@click.argument( + "roles", + required=True, +) +@click.argument( + "user-note", + required=False, +) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +@click.option( + "--add", + help="Add role to users", + is_flag=True, +) +@click.option( + "--remove", + help="Remove role from users", + is_flag=True, +) +@click.option( + "--update-notes", + help="Remove role from users", + is_flag=True, +) @click.option( "-rf", "--role-filter", - help='Comma seperated string to generate roles filter from eg "role1,role2,role3"', + help='Comma separated string to generate roles filter from eg "role1,role2,role3"', required=False, default="*", ) -@click.option("-uf", "--user-filter", help="Filter to find users", required=False, default="(userSector=*)") -def update_user_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, user_filter, role_filter): +@click.option( + "-uf", + "--user-filter", + help="Filter to find users", + required=False, + default="(userSector=*)", +) +def update_user_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter, + role_filter, +): cli.ldap.user.update_roles( roles, user_ou, @@ -59,16 +157,39 @@ def update_user_roles(roles, user_ou, root_dn, add, remove, update_notes, user_n @click.command() -@click.option("-t", "--rbac-repo-tag", help="RBAC repo tag to use", default="master") -def rbac_uplift(rbac_repo_tag): +@click.option( + "-t", + "--rbac-repo-tag", + help="RBAC repo tag to use", + default="master", +) +def rbac_uplift( + rbac_repo_tag, +): cli.ldap.rbac.main(rbac_repo_tag) @click.command() -@click.option("-u", "--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("-r", "--root-dn", help="Root DN to add users to, defaults to dc=moj,dc=com", default="dc=moj,dc=com") -def deactivate_crc_users(user_ou, root_dn): - cli.ldap.user.deactivate_crc_users(user_ou, root_dn) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def deactivate_crc_users( + user_ou, + root_dn, +): + cli.ldap.user.deactivate_crc_users( + user_ou, + root_dn, + ) # from cli.ldap import test diff --git a/cli/database/__init__.py b/cli/database/__init__.py index 9b64657..16b198d 100644 --- a/cli/database/__init__.py +++ b/cli/database/__init__.py @@ -1,6 +1,10 @@ import oracledb -from cli import env -from cli.logger import log +from cli import ( + env, +) +from cli.logger import ( + log, +) def connection(): diff --git a/cli/env.py b/cli/env.py index 2a96160..98c815f 100644 --- a/cli/env.py +++ b/cli/env.py @@ -1,6 +1,8 @@ import os -from dotenv import dotenv_values +from dotenv import ( + dotenv_values, +) import ast @@ -19,12 +21,28 @@ vars = { **{ - key.replace("VAR_", "").replace("_DICT", ""): ast.literal_eval(val) if "DICT" in key else val + key.replace( + "VAR_", + "", + ).replace( + "_DICT", + "", + ): ast.literal_eval(val) + if "DICT" in key + else val for key, val in dotenv_values(".vars").items() if val is not None }, # load development variables **{ - key.replace("VAR_", "").replace("_DICT", ""): ast.literal_eval(val) if "DICT" in key else val + key.replace( + "VAR_", + "", + ).replace( + "_DICT", + "", + ): ast.literal_eval(val) + if "DICT" in key + else val for key, val in os.environ.items() if key.startswith("VAR_") and val is not None }, @@ -32,14 +50,36 @@ # loads all environment variables starting with SECRET_ into a dictionary secrets = { **{ - key.replace("SECRET_", "").replace("_DICT", "").replace("SSM_", ""): ast.literal_eval(val) + key.replace( + "SECRET_", + "", + ) + .replace( + "_DICT", + "", + ) + .replace( + "SSM_", + "", + ): ast.literal_eval(val) if "_DICT" in key else val for key, val in dotenv_values(".secrets").items() if val is not None }, **{ - key.replace("SECRET_", "").replace("_DICT", "").replace("SSM_", ""): ast.literal_eval(val) + key.replace( + "SECRET_", + "", + ) + .replace( + "_DICT", + "", + ) + .replace( + "SSM_", + "", + ): ast.literal_eval(val) if "DICT" in key else val for key, val in os.environ.items() diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 2486329..48b9c1f 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -1,38 +1,78 @@ -from git import Repo +from git import ( + Repo, +) import jwt import time import requests import logging -def get_access_token(app_id, private_key, installation_id): +def get_access_token( + app_id, + private_key, + installation_id, +): # Create a JSON Web Token (JWT) using the app's private key now = int(time.time()) - payload = {"iat": now, "exp": now + 600, "iss": app_id} - jwt_token = jwt.encode(payload, private_key, algorithm="RS256") + payload = { + "iat": now, + "exp": now + 600, + "iss": app_id, + } + jwt_token = jwt.encode( + payload, + private_key, + algorithm="RS256", + ) # Exchange the JWT for an installation access token - headers = {"Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json"} + headers = { + "Authorization": f"Bearer {jwt_token}", + "Accept": "application/vnd.github.v3+json", + } response = requests.post( - f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers, ) # extract the token from the response access_token = response.json().get("token") return access_token -def get_repo(url, depth="1", branch_or_tag="master", token=None, auth_type="x-access-token", dest_name="repo"): +def get_repo( + url, + depth="1", + branch_or_tag="master", + token=None, + auth_type="x-access-token", + dest_name="repo", +): # if there is an @ in the url, assume auth is already specified - multi_options = ["--depth " + depth, "--branch " + branch_or_tag] + multi_options = [ + "--depth " + depth, + "--branch " + branch_or_tag, + ] if "@" in url: logging.info("auth already specified in url") - return Repo.clone_from(url, dest_name, multi_options=multi_options) + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) # if there is a token, assume auth is required and use the token and auth_type elif token: templated_url = f'https://{auth_type}:{token}@{url.split("//")[1]}' logging.info(f"cloning with token: {templated_url}") - return Repo.clone_from(templated_url, dest_name, multi_options=multi_options) + return Repo.clone_from( + templated_url, + dest_name, + multi_options=multi_options, + ) # if there is no token, assume auth is not required and clone without else: logging.info("cloning without auth") - return Repo.clone_from(url, dest_name, multi_options=multi_options) + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) diff --git a/cli/ldap/__init__.py b/cli/ldap/__init__.py index 07fe481..69110ed 100644 --- a/cli/ldap/__init__.py +++ b/cli/ldap/__init__.py @@ -1,12 +1,23 @@ -from ldap3 import Server, Connection, ALL +from ldap3 import ( + Server, + Connection, +) # import oracledb -def ldap_connect(ldap_host, ldap_user, ldap_password): +def ldap_connect( + ldap_host, + ldap_user, + ldap_password, +): server = Server(ldap_host) return Connection( - server=server, user=ldap_user, password=ldap_password, auto_bind="NO_TLS", authentication="SIMPLE" + server=server, + user=ldap_user, + password=ldap_password, + auto_bind="NO_TLS", + authentication="SIMPLE", ) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 240b090..339a6a6 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -1,14 +1,24 @@ import ldap3.utils.hashed -from cli.ldap import ldap_connect -from cli import env +from cli.ldap import ( + ldap_connect, +) +from cli import ( + env, +) import cli.git as git import glob -from cli.logger import log -from pathlib import Path +from cli.logger import ( + log, +) +from pathlib import ( + Path, +) import cli.template -from ldif import LDIFParser +from ldif import ( + LDIFParser, +) -#### example for token auth +# example for token auth # def get_repo_with_token(repo_tag="master"): # app_id = env.vars.get("GH_APP_ID") # private_key = env.vars.get("GH_PRIVATE_KEY") @@ -31,17 +41,26 @@ } -def get_repo(repo_tag="master"): +def get_repo( + repo_tag="master", +): url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" try: - repo = git.get_repo(url, dest_name="rbac", branch_or_tag=repo_tag) + repo = git.get_repo( + url, + dest_name="rbac", + branch_or_tag=repo_tag, + ) return repo except Exception as e: log.exception(e) return None -def prep_for_templating(files, strings=None): +def prep_for_templating( + files, + strings=None, +): rbac_substitutions = { "bind_password_hash.stdout": "bind_password_hash", r"ldap_config.base_users | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_users_ou", @@ -65,15 +84,33 @@ def prep_for_templating(files, strings=None): # print(type(strings)) for file_path in files: file = Path(file_path) - for k, v in strings.items(): - print("replacing", k, "with", v, "in", file_path) + for ( + k, + v, + ) in strings.items(): + print( + "replacing", + k, + "with", + v, + "in", + file_path, + ) file.write_text( - file.read_text().replace(k, v), + file.read_text().replace( + k, + v, + ), ) -def template_rbac(files): - hashed_pwd_admin_user = ldap3.utils.hashed.hashed(ldap3.HASHED_SALTED_SHA, env.secrets.get("LDAP_ADMIN_PASSWORD")) +def template_rbac( + files, +): + hashed_pwd_admin_user = ldap3.utils.hashed.hashed( + ldap3.HASHED_SALTED_SHA, + env.secrets.get("LDAP_ADMIN_PASSWORD"), + ) rendered_files = [] for file in files: @@ -86,22 +123,41 @@ def template_rbac(files): environment_name=env.vars.get("ENVIRONMENT_NAME"), project_name=env.vars.get("PROJECT_NAME"), ) - rendered_file = cli.template.save(rendered_text, file) + rendered_file = cli.template.save( + rendered_text, + file, + ) rendered_files.append(rendered_file) return rendered_files -def context_ldif(rendered_files): +def context_ldif( + rendered_files, +): context_file = [file for file in rendered_files if "context" in Path(file).name] for file in context_file: - parser = LDIFParser(open(file, "rb"), strict=False) - for dn, record in parser.parse(): + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) print(record) ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + ldap_connection.add( + dn, + attributes=record, ) - ldap_connection.add(dn, attributes=record) print(ldap_connection.result["result"]) if ldap_connection.result["result"] == 0: print(f"Successfully added context") @@ -113,25 +169,51 @@ def context_ldif(rendered_files): raise Exception(f"Failed to add {dn}... {record}") -def group_ldifs(rendered_files): +def group_ldifs( + rendered_files, +): # connect to ldap ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) group_files = [file for file in rendered_files if "groups" in Path(file).name] # loop through the group files for file in group_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) print(record) # add the record to ldap - ldap_connection.add(dn, attributes=record) + ldap_connection.add( + dn, + attributes=record, + ) if record.get("description"): print("updating description") - ldap_connection.modify(dn, {"description": [(ldap3.MODIFY_REPLACE, record["description"])]}) + ldap_connection.modify( + dn, + { + "description": [ + ( + ldap3.MODIFY_REPLACE, + record["description"], + ) + ] + }, + ) if ldap_connection.result["result"] == 0: print(f"Successfully added groups") elif ldap_connection.result["result"] == 68: @@ -142,10 +224,14 @@ def group_ldifs(rendered_files): raise Exception(f"Failed to add {dn}... {record}") -def policy_ldifs(rendered_files): +def policy_ldifs( + rendered_files, +): # connect to ldap ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) policy_files = [file for file in rendered_files if "policy" in Path(file).name] @@ -156,13 +242,25 @@ def policy_ldifs(rendered_files): # loop through the policy files for file in policy_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) # print(record) # add the record to ldap - ldap_connection.add(dn, attributes=record) + ldap_connection.add( + dn, + attributes=record, + ) if ldap_connection.result["result"] == 0: print(f"Successfully added policies") elif ldap_connection.result["result"] == 68: @@ -173,10 +271,14 @@ def policy_ldifs(rendered_files): raise Exception(f"Failed to add {dn}... {record}") -def role_ldifs(rendered_files): +def role_ldifs( + rendered_files, +): # connect to ldap ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) role_files = [file for file in rendered_files if "nd_role" in Path(file).name] @@ -191,13 +293,25 @@ def role_ldifs(rendered_files): # loop through the role files for file in role_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) # print(record) # add the record to ldap - ldap_connection.add(dn, attributes=record) + ldap_connection.add( + dn, + attributes=record, + ) if ldap_connection.result["result"] == 0: print(f"Successfully added roles") elif ldap_connection.result["result"] == 68: @@ -210,24 +324,44 @@ def role_ldifs(rendered_files): # not complete!! # see https://github.com/ministryofjustice/hmpps-delius-pipelines/blob/master/components/delius-core/playbooks/rbac/import_schemas.yml -def schema_ldifs(rendered_files): +def schema_ldifs( + rendered_files, +): # connect to ldap ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) - schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] + schema_files = [ + file + for file in rendered_files + if "delius.ldif" or "pwm.ldif" in Path(file).name + ] # loop through the schema files for file in schema_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) # print(record) # add the record to ldap - ldap_connection.add(dn, attributes=record) + ldap_connection.add( + dn, + attributes=record, + ) if ldap_connection.result["result"] == 0: print(f"Successfully added schemas") elif ldap_connection.result["result"] == 68: @@ -238,23 +372,37 @@ def schema_ldifs(rendered_files): raise Exception(f"Failed to add {dn}... {record}") -def user_ldifs(rendered_files): +def user_ldifs( + rendered_files, +): # connect to ldap ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) user_files = [file for file in rendered_files if "-users" in Path(file).name] # first, delete the users for file in user_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for (dn,) in parser.parse(): print("got entry record: %s" % dn) # for each user find child entries and delete them - ldap_connection.search(dn, "(objectclass=*)", search_scope=ldap3.SUBTREE) + ldap_connection.search( + dn, + "(objectclass=*)", + search_scope=ldap3.SUBTREE, + ) for entry in ldap_connection.entries: print(entry.entry_dn) ldap_connection.delete(entry.entry_dn) @@ -266,13 +414,25 @@ def user_ldifs(rendered_files): # loop through the user files for file in user_files: # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) + parser = LDIFParser( + open( + file, + "rb", + ), + strict=False, + ) # loop through the records - for dn, record in parser.parse(): + for ( + dn, + record, + ) in parser.parse(): print("got entry record: %s" % dn) # print(record) # add the record to ldap - ldap_connection.add(dn, attributes=record) + ldap_connection.add( + dn, + attributes=record, + ) if ldap_connection.result["result"] == 0: print(f"Successfully added users") elif ldap_connection.result["result"] == 68: @@ -283,12 +443,20 @@ def user_ldifs(rendered_files): raise Exception(f"Failed to add {dn}... {record}") -def main(rbac_repo_tag, clone_path="./rbac"): - repo = get_repo(rbac_repo_tag) +def main( + rbac_repo_tag, + clone_path="./rbac", +): + get_repo(rbac_repo_tag) files = [ file - for file in glob.glob(f"{clone_path}/**/*", recursive=True) - if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") + for file in glob.glob( + f"{clone_path}/**/*", + recursive=True, + ) + if Path(file).is_file() + and Path(file).name.endswith(".ldif") + or Path(file).name.endswith(".j2") ] prep_for_templating(files) diff --git a/cli/ldap/user.py b/cli/ldap/user.py index 3fb24ea..d401cf3 100644 --- a/cli/ldap/user.py +++ b/cli/ldap/user.py @@ -2,43 +2,84 @@ import cli.ldap -from cli.logger import log -from cli import env - -from cli.ldap import ldap_connect -from ldap3 import MODIFY_REPLACE, DEREF_NEVER +from cli.logger import ( + log, +) +from cli import ( + env, +) + +from cli.ldap import ( + ldap_connect, +) +from ldap3 import ( + MODIFY_REPLACE, + DEREF_NEVER, +) import cli.database -from itertools import product +from itertools import ( + product, +) -from datetime import datetime +from datetime import ( + datetime, +) ######################################### # Change a users home area ######################################### -def change_home_areas(old_home_area, new_home_area, user_ou, root_dn, attribute="userHomeArea", object_class="NDUser"): +def change_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, + attribute="userHomeArea", + object_class="NDUser", +): log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) - search_filter = ( - f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + search_filter = f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + ldap_connection.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + search_filter, + attributes=[attribute], ) - ldap_connection.search(",".join([user_ou, root_dn]), search_filter, attributes=[attribute]) # Iterate through the search results and update the attribute for entry in ldap_connection.entries: dn = entry.entry_dn - changes = {attribute: [(MODIFY_REPLACE, [new_home_area])]} - ldap_connection.modify(dn, changes) + changes = { + attribute: [ + ( + MODIFY_REPLACE, + [new_home_area], + ) + ] + } + ldap_connection.modify( + dn, + changes, + ) # Check if the modification was successful if ldap_connection.result["result"] == 0: log.info(f"Successfully updated {attribute} for {dn}") else: - log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") + log.error( + f"Failed to update {attribute} for {dn}: {ldap_connection.result}" + ) ######################################### @@ -46,25 +87,40 @@ def change_home_areas(old_home_area, new_home_area, user_ou, root_dn, attribute= ######################################### -def parse_user_role_list(user_role_list): +def parse_user_role_list( + user_role_list, +): # The format of the list should be a pipe separated list of username and role lists, # where the username and role list is separated by a comma character, # and the roles are separated by a semi-colon: # username1,role1;role2;role3|username2,role1;role2 - return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} + return { + user.split(",")[0]: user.split(",")[1].split(";") + for user in user_role_list.split("|") + } -def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): +def add_roles_to_user( + username, + roles, + user_ou="ou=Users", + root_dn="dc=moj,dc=com", +): log.info(f"Adding roles {roles} to user {username}") ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) for role in roles: ldap_connection.add( f"cn={role},cn={username},{user_ou},{root_dn}", attributes={ - "objectClass": ["NDRoleAssociation", "alias"], + "objectClass": [ + "NDRoleAssociation", + "alias", + ], "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", }, ) @@ -78,11 +134,23 @@ def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=co raise Exception(f"Failed to add role {role} to user {username}") -def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): +def process_user_roles_list( + user_role_list, + user_ou="ou=Users", + root_dn="dc=moj,dc=com", +): log.info(f"secrets: {env.secrets}") user_roles = parse_user_role_list(user_role_list) - for user, roles in user_roles.items(): - add_roles_to_user(user, roles, user_ou, root_dn) + for ( + user, + roles, + ) in user_roles.items(): + add_roles_to_user( + user, + roles, + user_ou, + root_dn, + ) ######################################### @@ -105,12 +173,29 @@ def update_roles( log.error("User note must be provided when updating notes") raise Exception("User note must be provided when updating notes") ldap_connection_user_filter = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) # # Search for users matching the user_filter - ldap_connection_user_filter.search(",".join([user_ou, root_dn]), user_filter, attributes=["cn"]) - users_found = sorted([entry.cn.value for entry in ldap_connection_user_filter.entries if entry.cn.value]) + ldap_connection_user_filter.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + user_filter, + attributes=["cn"], + ) + users_found = sorted( + [ + entry.cn.value + for entry in ldap_connection_user_filter.entries + if entry.cn.value + ] + ) log.debug("users found from user filter") log.debug(users_found) ldap_connection_user_filter.unbind() @@ -120,25 +205,35 @@ def update_roles( # create role filter if len(roles_filter_list) > 0: - full_role_filter = ( - f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" - ) + full_role_filter = f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" else: full_role_filter = "(&(objectclass=NDRoleAssociation)(cn=*))" # Search for roles matching the role_filter ldap_connection_role_filter = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) ldap_connection_role_filter.search( - ",".join([user_ou, root_dn]), + ",".join( + [ + user_ou, + root_dn, + ] + ), full_role_filter, attributes=["cn"], dereference_aliases=DEREF_NEVER, ) roles_found = sorted( - list(set([entry.entry_dn.split(",")[1].split("=")[1] for entry in ldap_connection_role_filter.entries])) + set( + { + entry.entry_dn.split(",")[1].split("=")[1] + for entry in ldap_connection_role_filter.entries + } + ) ) log.debug("users found from roles filter: ") log.debug(roles_found) @@ -152,12 +247,19 @@ def update_roles( # cartesian_product = [(user, role) for user in matched_users for role in roles] - cartesian_product = list(product(matched_users, roles)) + cartesian_product = list( + product( + matched_users, + roles, + ) + ) log.debug("cartesian product: ") log.debug(cartesian_product) ldap_connection_action = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) for item in cartesian_product: @@ -167,7 +269,11 @@ def update_roles( attributes={ "cn": item[1], "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", - "objectClass": ["NDRoleAssociation", "alias", "top"], + "objectClass": [ + "NDRoleAssociation", + "alias", + "top", + ], }, ) if ldap_connection_action.result["result"] == 0: @@ -178,7 +284,9 @@ def update_roles( log.error(f"Failed to add role '{item[1]}' to user '{item[0]}'") log.debug(ldap_connection_action.result) elif remove: - ldap_connection_action.delete(f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}") + ldap_connection_action.delete( + f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}" + ) if ldap_connection_action.result["result"] == 0: log.info(f"Successfully removed role '{item[1]}' from user '{item[0]}'") elif ldap_connection_action.result["result"] == 32: @@ -194,9 +302,15 @@ def update_roles( log.debug("Created database cursor successfully") for user in matched_users: try: - update_sql = "UPDATE USER_ SET LAST_UPDATED_DATETIME=CURRENT_DATE, LAST_UPDATED_USER_ID=4 WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + update_sql = """ + UPDATE USER_ SET LAST_UPDATED_DATETIME=CURRENT_DATE, + LAST_UPDATED_USER_ID=4 WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn) + """ update_cursor = connection.cursor() - update_cursor.execute(update_sql, [user]) + update_cursor.execute( + update_sql, + [user], + ) update_cursor.close() insert_sql = """ @@ -220,7 +334,11 @@ def update_roles( """ insert_cursor = connection.cursor() insert_cursor.setinputsizes(user_note=oracledb.CLOB) - insert_cursor.execute(insert_sql, user_note=user_note, user_dn=user) + insert_cursor.execute( + insert_sql, + user_note=user_note, + user_dn=user, + ) insert_cursor.close() log.info(f"Updated notes for user {user}") @@ -236,13 +354,20 @@ def update_roles( ######################################### -def deactivate_crc_users(user_ou, root_dn): +def deactivate_crc_users( + user_ou, + root_dn, +): log.info("Deactivating CRC users") ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) - user_filter = "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + user_filter = ( + "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + ) home_areas = [ [ @@ -273,7 +398,12 @@ def deactivate_crc_users(user_ou, root_dn): found_users = [] for home_area in home_areas: ldap_connection.search( - ",".join([user_ou, root_dn]), + ",".join( + [ + user_ou, + root_dn, + ] + ), f"(&(userHomeArea={home_area})(!(cn={home_area})){user_filter})", attributes=["dn"], ) @@ -281,7 +411,12 @@ def deactivate_crc_users(user_ou, root_dn): found_users.append(entry.entry_dn for entry in ldap_connection.entries) ldap_connection.search( - ",".join([user_ou, root_dn]), + ",".join( + [ + user_ou, + root_dn, + ] + ), f"(&(!(userHomeArea=*)){user_filter})", attributes=["dn"], ) @@ -292,16 +427,27 @@ def deactivate_crc_users(user_ou, root_dn): date_str = f"{datetime.now().strftime('%Y%m%d')}000000Z" for user in all_users: - ldap_connection.modify(user, {"endDate": [(MODIFY_REPLACE, [date_str])]}) + ldap_connection.modify( + user, + { + "endDate": [ + ( + MODIFY_REPLACE, + [date_str], + ) + ] + }, + ) connection = cli.database.connection() for user_dn in all_users: try: - update_sql = ( - f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" - ) + update_sql = f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" update_cursor = connection.cursor() - update_cursor.execute(update_sql, [user_dn]) + update_cursor.execute( + update_sql, + [user_dn], + ) update_cursor.close() log.info(f"Updated END_DATE for user {user_dn}") connection.commit() diff --git a/cli/logger.py b/cli/logger.py index aba6e5f..f4c84f1 100644 --- a/cli/logger.py +++ b/cli/logger.py @@ -6,25 +6,46 @@ def configure_logging(): class SensitiveFormatter(logging.Formatter): """Formatter that removes secrets from log messages.""" - def __init__(self, format_str=None, datefmt_str=None): - super().__init__(fmt=format_str, datefmt=datefmt_str) - self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here + def __init__( + self, + format_str=None, + datefmt_str=None, + ): + super().__init__( + fmt=format_str, + datefmt=datefmt_str, + ) + self._secrets_set = set( + cli.env.secrets.values() + ) # Retrieve secrets set here self.default_msec_format = "%s.%03d" - def _filter(self, s): + def _filter( + self, + s, + ): redacted = " ".join( - ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] + [ + "*" * len(string) if string in self._secrets_set else string + for string in s.split(" ") + ] ) return redacted - def format(self, record): + def format( + self, + record, + ): original = super().format(record) return self._filter(original) print("configure_logging") """Configure logging based on environment variables.""" - format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + format = ( + cli.env.vars.get("LOG_FORMAT") + or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + ) datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" log = logging.getLogger(__name__) @@ -33,7 +54,12 @@ def format(self, record): logging.root.handlers = [] handler = logging.StreamHandler() - handler.setFormatter(SensitiveFormatter(format_str=format, datefmt_str=datefmt)) + handler.setFormatter( + SensitiveFormatter( + format_str=format, + datefmt_str=datefmt, + ) + ) logging.root.addHandler(handler) if cli.env.vars.get("LOG_LEVEL") == "DEBUG": print("DEBUG") diff --git a/cli/template/__init__.py b/cli/template/__init__.py index 55c42c1..0a02f78 100644 --- a/cli/template/__init__.py +++ b/cli/template/__init__.py @@ -1,24 +1,55 @@ import os.path import jinja2 -from pathlib import Path +from pathlib import ( + Path, +) def render(template_path, **kwargs): parent_path = Path(template_path).parent - env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=parent_path), autoescape=True) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=parent_path), + autoescape=True, + ) template = env.get_template(Path(template_path).name) return template.render(**kwargs) -def save(rendered_text, template_path, rendered_dir="./rendered/"): +def save( + rendered_text, + template_path, + rendered_dir="./rendered/", +): # create rendered_dir if it doesn't exist if not Path.exists(Path(rendered_dir)): Path.mkdir(Path(rendered_dir)) # create the directory structure for the template file if it doesn't exist - if not Path.exists(Path(os.path.join(rendered_dir, template_path)).parent): - Path.mkdir(Path(os.path.join(rendered_dir, template_path)).parent) - file = Path(os.path.join(rendered_dir, template_path.replace(".j2", ""))) + if not Path.exists( + Path( + os.path.join( + rendered_dir, + template_path, + ) + ).parent + ): + Path.mkdir( + Path( + os.path.join( + rendered_dir, + template_path, + ) + ).parent + ) + file = Path( + os.path.join( + rendered_dir, + template_path.replace( + ".j2", + "", + ), + ) + ) file.touch(exist_ok=True) file.write_text(rendered_text) return file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..67caa7b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +# PEP-8 line length is not very practical +line-length = 88 +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.py_cache + | \venv +)/ +''' + +[tool.codespell] +skip = './venv,./rbac,./rendered' +count = '' +quiet-level = 3 \ No newline at end of file diff --git a/readme.md b/readme.md index eaa445a..3475378 100644 --- a/readme.md +++ b/readme.md @@ -1,9 +1,9 @@ # Quick start guide - ## Environment variables -Variables are picked up from the environment, or can be specified in a `.env` file in the current directory (at the same level as the file `setup.py`) +Variables are picked up from the environment, or can be specified in a `.env` file in the current directory (at the same +level as the file `setup.py`) See `cli/config.py` for a list of variables. ## Installation for development purposes @@ -20,10 +20,20 @@ See `cli/config.py` for a list of variables. `pip install git+https://github.com/ministryofjustice/hmpps-ldap-automation-cli.git` -Optionally append `@`, `@` or -`@` to the end of the url to install a specific +Optionally append `@`, `@` or +`@` to the end of the url to install a specific commit ## Usage `ldap-automation --help` + +# Dev + +## pre-commit + +This project uses pre-commit to run a number of checks on the code before it is committed. +To install the pre-commit hooks run: + +`pre-commit install` + diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..ae204bd --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,14 @@ +click==8.1.6 +ldap3 +oracledb==1.4 +ansible-runner +PyGithub +GitPython +pyjwt +python-dotenv +Jinja2 +git+https://github.com/abilian/ldif.git@4.2.0 +pre-commit +black +flake8 +codespell \ No newline at end of file diff --git a/setup.py b/setup.py index cd85c83..b28c4dc 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,7 @@ -from setuptools import setup, find_packages +from setuptools import ( + setup, + find_packages, +) # Read requirements from requirements.txt with open("requirements.txt") as f: @@ -6,7 +9,9 @@ standard_pkgs = [r for r in requirements if not r.startswith("git+")] git_pkgs = [r for r in requirements if r.startswith("git+")] -formatted_git_pkgs = [f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs] +formatted_git_pkgs = [ + f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs +] all_reqs = standard_pkgs + formatted_git_pkgs setup( From f8620b40205cddd2abd285a6ad3ea60267a800a9 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 25 Sep 2023 16:02:55 +0100 Subject: [PATCH 22/29] update logging and requirements --- cli/logger.py | 18 +++++++----------- requirements.txt | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cli/logger.py b/cli/logger.py index f4c84f1..16afdf1 100644 --- a/cli/logger.py +++ b/cli/logger.py @@ -15,9 +15,7 @@ def __init__( fmt=format_str, datefmt=datefmt_str, ) - self._secrets_set = set( - cli.env.secrets.values() - ) # Retrieve secrets set here + self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here self.default_msec_format = "%s.%03d" def _filter( @@ -25,10 +23,7 @@ def _filter( s, ): redacted = " ".join( - [ - "*" * len(string) if string in self._secrets_set else string - for string in s.split(" ") - ] + ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] ) return redacted @@ -42,10 +37,7 @@ def format( print("configure_logging") """Configure logging based on environment variables.""" - format = ( - cli.env.vars.get("LOG_FORMAT") - or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" - ) + format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" log = logging.getLogger(__name__) @@ -73,6 +65,10 @@ def format( elif cli.env.vars.get("LOG_LEVEL") == "INFO": print("INFO") log.setLevel(logging.INFO) + else: + print("No LOG_LEVEL set, defaulting to INFO") + print("INFO") + log.setLevel(logging.INFO) return True diff --git a/requirements.txt b/requirements.txt index 3888448..79a4e4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ ldap3 oracledb==1.4 ansible-runner PyGithub -GitPython +GitPython==3.1.37 pyjwt python-dotenv Jinja2 From 3903203b274bf0d6496e7a8a0dfc7d5cd506b3a2 Mon Sep 17 00:00:00 2001 From: Andrew Moore Date: Fri, 29 Sep 2023 13:15:46 +0100 Subject: [PATCH 23/29] NIT-854 Add exception handling and add logging where appropriate --- cli/database/__init__.py | 4 +- cli/git/__init__.py | 63 +++++--- cli/ldap/rbac.py | 317 +++++++++++++++++++++++---------------- cli/ldap/user.py | 179 +++++++++++++--------- cli/logger.py | 14 +- 5 files changed, 356 insertions(+), 221 deletions(-) diff --git a/cli/database/__init__.py b/cli/database/__init__.py index 16b198d..511366b 100644 --- a/cli/database/__init__.py +++ b/cli/database/__init__.py @@ -13,5 +13,7 @@ def connection(): log.debug("Created database connection successfully") return conn except Exception as e: - log.exception(e) + log.exception( + f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}" + ) raise e diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 48b9c1f..e79dae8 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -30,10 +30,17 @@ def get_access_token( "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.post( - f"https://api.github.com/app/installations/{installation_id}/access_tokens", - headers=headers, - ) + try: + response = requests.post( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers, + ) + except Exception as e: + logging.exception( + f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}" + ) + raise (e) + # extract the token from the response access_token = response.json().get("token") return access_token @@ -54,25 +61,43 @@ def get_repo( ] if "@" in url: logging.info("auth already specified in url") - return Repo.clone_from( - url, - dest_name, - multi_options=multi_options, - ) + try: + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise (e) # if there is a token, assume auth is required and use the token and auth_type elif token: templated_url = f'https://{auth_type}:{token}@{url.split("//")[1]}' logging.info(f"cloning with token: {templated_url}") - return Repo.clone_from( - templated_url, - dest_name, - multi_options=multi_options, - ) + try: + return Repo.clone_from( + templated_url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise (e) # if there is no token, assume auth is not required and clone without else: logging.info("cloning without auth") - return Repo.clone_from( - url, - dest_name, - multi_options=multi_options, - ) + try: + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise (e) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 339a6a6..5e8c9f6 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -79,23 +79,14 @@ def prep_for_templating( if strings is None: strings = env.vars.get("RBAC_SUBSTITUTIONS") or rbac_substitutions - # get a list of files - # print(strings) - # print(type(strings)) + for file_path in files: file = Path(file_path) for ( k, v, ) in strings.items(): - print( - "replacing", - k, - "with", - v, - "in", - file_path, - ) + log.info(f"replacing {k} with {v} in {file_path}") file.write_text( file.read_text().replace( k, @@ -147,25 +138,35 @@ def context_ldif( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - print(record) - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - ldap_connection.add( - dn, - attributes=record, - ) - print(ldap_connection.result["result"]) + log.info(f"got entry record: {dn}") + log.debug(record) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + try: + ldap_connection.add( + dn, + attributes=record, + ) + log.debug(ldap_connection.result["result"]) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added context") + log.info("successfully added context") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") @@ -173,11 +174,16 @@ def group_ldifs( rendered_files, ): # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + group_files = [file for file in rendered_files if "groups" in Path(file).name] # loop through the group files for file in group_files: @@ -194,33 +200,43 @@ def group_ldifs( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - print(record) + log.debug(f"got entry record: {dn}") + log.debug(record) # add the record to ldap - ldap_connection.add( - dn, - attributes=record, - ) - if record.get("description"): - print("updating description") - ldap_connection.modify( + try: + ldap_connection.add( dn, - { - "description": [ - ( - ldap3.MODIFY_REPLACE, - record["description"], - ) - ] - }, + attributes=record, ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + + if record.get("description"): + log.info(f"Updating description for {record}") + try: + ldap_connection.modify( + dn, + { + "description": [ + ( + ldap3.MODIFY_REPLACE, + record["description"], + ) + ] + }, + ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added groups") + log.info(f"Successfully added groups") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") @@ -228,11 +244,16 @@ def policy_ldifs( rendered_files, ): # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + policy_files = [file for file in rendered_files if "policy" in Path(file).name] # first, delete the policies @@ -254,20 +275,24 @@ def policy_ldifs( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - # print(record) + log.info(f"got entry record: {dn}") # add the record to ldap - ldap_connection.add( - dn, - attributes=record, - ) + try: + ldap_connection.add( + dn, + attributes=record, + ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added policies") + log.info(f"Successfully added policies") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") @@ -275,11 +300,15 @@ def role_ldifs( rendered_files, ): # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e role_files = [file for file in rendered_files if "nd_role" in Path(file).name] # first, delete the roles @@ -305,20 +334,24 @@ def role_ldifs( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - # print(record) + log.info(f"got entry record: {dn}") # add the record to ldap - ldap_connection.add( - dn, - attributes=record, - ) + try: + ldap_connection.add( + dn, + attributes=record, + ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added roles") + log.info(f"Successfully added roles") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") @@ -328,11 +361,15 @@ def schema_ldifs( rendered_files, ): # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e schema_files = [ file @@ -355,20 +392,24 @@ def schema_ldifs( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - # print(record) + log.info(f"got entry record: {dn}") # add the record to ldap - ldap_connection.add( - dn, - attributes=record, - ) + try: + ldap_connection.add( + dn, + attributes=record, + ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added schemas") + log.info(f"Successfully added schemas") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") @@ -376,11 +417,16 @@ def user_ldifs( rendered_files, ): # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise (e) + user_files = [file for file in rendered_files if "-users" in Path(file).name] # first, delete the users @@ -395,21 +441,33 @@ def user_ldifs( ) # loop through the records for (dn,) in parser.parse(): - print("got entry record: %s" % dn) + log.info(f"got entry record: {dn}") - # for each user find child entries and delete them - ldap_connection.search( - dn, - "(objectclass=*)", - search_scope=ldap3.SUBTREE, - ) - for entry in ldap_connection.entries: - print(entry.entry_dn) - ldap_connection.delete(entry.entry_dn) - - # print(record) - # add the record to ldap - ldap_connection.delete(dn) + # for each user find child entries + try: + ldap_connection.search( + dn, + "(objectclass=*)", + search_scope=ldap3.SUBTREE, + ) + except Exception as e: + log.exception(f"Failed to search {dn}") + raise (e) + + # delete child entries + try: + for entry in ldap_connection.entries: + log.debug(entry.entry_dn) + ldap_connection.delete(entry.entry_dn) + except Exception as e: + log.exception(f"Failed to delete {entry.entry_dn}") + raise (e) + + try: + ldap_connection.delete(dn) + except Exception as e: + log.exception(f"Failed to delete {dn}") + raise (e) # loop through the user files for file in user_files: @@ -426,20 +484,25 @@ def user_ldifs( dn, record, ) in parser.parse(): - print("got entry record: %s" % dn) - # print(record) + log.info(f"got entry record: {dn}") + # add the record to ldap - ldap_connection.add( - dn, - attributes=record, - ) + try: + ldap_connection.add( + dn, + attributes=record, + ) + except Exception as e: + log.exception(f"Failed to add {dn}... {record}") + raise (e) + if ldap_connection.result["result"] == 0: - print(f"Successfully added users") + log.info(f"Successfully added users") elif ldap_connection.result["result"] == 68: - print(f"{dn} already exists") + log.info(f"{dn} already exists") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add {dn}... {record}") diff --git a/cli/ldap/user.py b/cli/ldap/user.py index d401cf3..1aa4efa 100644 --- a/cli/ldap/user.py +++ b/cli/ldap/user.py @@ -114,23 +114,28 @@ def add_roles_to_user( env.secrets.get("LDAP_BIND_PASSWORD"), ) for role in roles: - ldap_connection.add( - f"cn={role},cn={username},{user_ou},{root_dn}", - attributes={ - "objectClass": [ - "NDRoleAssociation", - "alias", - ], - "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", - }, - ) + try: + ldap_connection.add( + f"cn={role},cn={username},{user_ou},{root_dn}", + attributes={ + "objectClass": [ + "NDRoleAssociation", + "alias", + ], + "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", + }, + ) + except Exception as e: + log.exception(f"Failed to add role {role} to user {username}") + raise e + if ldap_connection.result["result"] == 0: - print(f"Successfully added role {role} to user {username}") + log.info(f"Successfully added role {role} to user {username}") elif ldap_connection.result["result"] == 68: - print(f"Role {role} already exists for user {username}") + log.info(f"Role {role} already exists for user {username}") else: - print(ldap_connection.result) - print(ldap_connection.response) + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) raise Exception(f"Failed to add role {role} to user {username}") @@ -141,16 +146,20 @@ def process_user_roles_list( ): log.info(f"secrets: {env.secrets}") user_roles = parse_user_role_list(user_role_list) - for ( - user, - roles, - ) in user_roles.items(): - add_roles_to_user( + try: + for ( user, roles, - user_ou, - root_dn, - ) + ) in user_roles.items(): + add_roles_to_user( + user, + roles, + user_ou, + root_dn, + ) + except Exception as e: + log.exception(f"Failed to add role to user") + raise e ######################################### @@ -172,23 +181,33 @@ def update_roles( if update_notes and (user_note is None or len(user_note) < 1): log.error("User note must be provided when updating notes") raise Exception("User note must be provided when updating notes") - ldap_connection_user_filter = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + + try: + ldap_connection_user_filter = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e # # Search for users matching the user_filter - ldap_connection_user_filter.search( - ",".join( - [ - user_ou, - root_dn, - ] - ), - user_filter, - attributes=["cn"], - ) + try: + ldap_connection_user_filter.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + user_filter, + attributes=["cn"], + ) + except Exception as e: + log.exception("Failed to search for users") + raise e + users_found = sorted( [ entry.cn.value @@ -210,23 +229,33 @@ def update_roles( full_role_filter = "(&(objectclass=NDRoleAssociation)(cn=*))" # Search for roles matching the role_filter - ldap_connection_role_filter = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - ldap_connection_role_filter.search( - ",".join( - [ - user_ou, - root_dn, - ] - ), - full_role_filter, - attributes=["cn"], - dereference_aliases=DEREF_NEVER, - ) + try: + ldap_connection_role_filter = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e + + try: + ldap_connection_role_filter.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + full_role_filter, + attributes=["cn"], + dereference_aliases=DEREF_NEVER, + ) + except Exception as e: + log.exception("Failed to search for roles") + raise e + roles_found = sorted( set( { @@ -256,32 +285,40 @@ def update_roles( log.debug("cartesian product: ") log.debug(cartesian_product) - ldap_connection_action = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) + try: + ldap_connection_action = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e for item in cartesian_product: if add: - ldap_connection_action.add( - f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}", - attributes={ - "cn": item[1], - "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", - "objectClass": [ - "NDRoleAssociation", - "alias", - "top", - ], - }, - ) + try: + ldap_connection_action.add( + f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}", + attributes={ + "cn": item[1], + "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", + "objectClass": [ + "NDRoleAssociation", + "alias", + "top", + ], + }, + ) + except Exception as e: + log.exception(f"Failed to add role '{item[1]}' to user '{item[0]}'") + raise e if ldap_connection_action.result["result"] == 0: log.info(f"Successfully added role '{item[1]}' to user '{item[0]}'") elif ldap_connection_action.result["result"] == 68: log.info(f"Role '{item[1]}' already present for user '{item[0]}'") else: - log.error(f"Failed to add role '{item[1]}' to user '{item[0]}'") + log.e(f"Failed to add role '{item[1]}' to user '{item[0]}'") log.debug(ldap_connection_action.result) elif remove: ldap_connection_action.delete( diff --git a/cli/logger.py b/cli/logger.py index 16afdf1..a76bb6a 100644 --- a/cli/logger.py +++ b/cli/logger.py @@ -15,7 +15,9 @@ def __init__( fmt=format_str, datefmt=datefmt_str, ) - self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here + self._secrets_set = set( + cli.env.secrets.values() + ) # Retrieve secrets set here self.default_msec_format = "%s.%03d" def _filter( @@ -23,7 +25,10 @@ def _filter( s, ): redacted = " ".join( - ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] + [ + "*" * len(string) if string in self._secrets_set else string + for string in s.split(" ") + ] ) return redacted @@ -37,7 +42,10 @@ def format( print("configure_logging") """Configure logging based on environment variables.""" - format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + format = ( + cli.env.vars.get("LOG_FORMAT") + or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + ) datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" log = logging.getLogger(__name__) From 1deeb0ab02f8026a18151a53be86182244d2eeed Mon Sep 17 00:00:00 2001 From: Andrew Moore Date: Fri, 29 Sep 2023 16:51:40 +0100 Subject: [PATCH 24/29] NIT-854 fix typos --- cli/git/__init__.py | 8 ++++---- cli/ldap/rbac.py | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cli/git/__init__.py b/cli/git/__init__.py index e79dae8..8a733c9 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -39,7 +39,7 @@ def get_access_token( logging.exception( f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}" ) - raise (e) + raise e # extract the token from the response access_token = response.json().get("token") @@ -71,7 +71,7 @@ def get_repo( logging.exception( f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" ) - raise (e) + raise e # if there is a token, assume auth is required and use the token and auth_type elif token: templated_url = f'https://{auth_type}:{token}@{url.split("//")[1]}' @@ -86,7 +86,7 @@ def get_repo( logging.exception( f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" ) - raise (e) + raise e # if there is no token, assume auth is not required and clone without else: logging.info("cloning without auth") @@ -100,4 +100,4 @@ def get_repo( logging.exception( f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" ) - raise (e) + raise e diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 5e8c9f6..52510cc 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -158,7 +158,7 @@ def context_ldif( log.debug(ldap_connection.result["result"]) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info("successfully added context") @@ -210,7 +210,7 @@ def group_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if record.get("description"): log.info(f"Updating description for {record}") @@ -228,7 +228,7 @@ def group_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info(f"Successfully added groups") @@ -284,7 +284,7 @@ def policy_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info(f"Successfully added policies") @@ -343,7 +343,7 @@ def role_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info(f"Successfully added roles") @@ -401,7 +401,7 @@ def schema_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info(f"Successfully added schemas") @@ -425,7 +425,7 @@ def user_ldifs( ) except Exception as e: log.exception(f"Failed to connect to ldap") - raise (e) + raise e user_files = [file for file in rendered_files if "-users" in Path(file).name] @@ -452,7 +452,7 @@ def user_ldifs( ) except Exception as e: log.exception(f"Failed to search {dn}") - raise (e) + raise e # delete child entries try: @@ -461,13 +461,13 @@ def user_ldifs( ldap_connection.delete(entry.entry_dn) except Exception as e: log.exception(f"Failed to delete {entry.entry_dn}") - raise (e) + raise e try: ldap_connection.delete(dn) except Exception as e: log.exception(f"Failed to delete {dn}") - raise (e) + raise e # loop through the user files for file in user_files: @@ -494,7 +494,7 @@ def user_ldifs( ) except Exception as e: log.exception(f"Failed to add {dn}... {record}") - raise (e) + raise e if ldap_connection.result["result"] == 0: log.info(f"Successfully added users") From 1d94eab6806cd606eda8fb101fccfeedde69fe5b Mon Sep 17 00:00:00 2001 From: Andrew Moore <20435317+andrewmooreio@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:11:00 +0100 Subject: [PATCH 25/29] Apply suggestions from code review Co-authored-by: George Taylor --- cli/ldap/rbac.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 52510cc..63becc7 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -82,11 +82,12 @@ def prep_for_templating( for file_path in files: file = Path(file_path) + log.info("Replacing strings in rbac files") for ( k, v, ) in strings.items(): - log.info(f"replacing {k} with {v} in {file_path}") + log.debug(f"replacing {k} with {v} in {file_path}") file.write_text( file.read_text().replace( k, @@ -161,7 +162,7 @@ def context_ldif( raise e if ldap_connection.result["result"] == 0: - log.info("successfully added context") + log.info("Successfully added context") elif ldap_connection.result["result"] == 68: log.info(f"{dn} already exists") else: @@ -275,7 +276,7 @@ def policy_ldifs( dn, record, ) in parser.parse(): - log.info(f"got entry record: {dn}") + log.info(f"Got entry record: {dn}") # add the record to ldap try: ldap_connection.add( @@ -334,7 +335,7 @@ def role_ldifs( dn, record, ) in parser.parse(): - log.info(f"got entry record: {dn}") + log.info(f"Got entry record: {dn}") # add the record to ldap try: ldap_connection.add( @@ -392,7 +393,7 @@ def schema_ldifs( dn, record, ) in parser.parse(): - log.info(f"got entry record: {dn}") + log.info(f"Got entry record: {dn}") # add the record to ldap try: ldap_connection.add( @@ -441,7 +442,7 @@ def user_ldifs( ) # loop through the records for (dn,) in parser.parse(): - log.info(f"got entry record: {dn}") + log.info(f"Got entry record: {dn}") # for each user find child entries try: @@ -484,7 +485,7 @@ def user_ldifs( dn, record, ) in parser.parse(): - log.info(f"got entry record: {dn}") + log.info(f"Got entry record: {dn}") # add the record to ldap try: From ad0303bf8c1039a33d597a35210a20bf8202c953 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 20 Nov 2023 13:44:44 +0000 Subject: [PATCH 26/29] Update rbac.py --- cli/ldap/rbac.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py index 63becc7..c7b6204 100644 --- a/cli/ldap/rbac.py +++ b/cli/ldap/rbac.py @@ -372,11 +372,7 @@ def schema_ldifs( log.exception(f"Failed to connect to ldap") raise e - schema_files = [ - file - for file in rendered_files - if "delius.ldif" or "pwm.ldif" in Path(file).name - ] + schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] # loop through the schema files for file in schema_files: @@ -441,7 +437,7 @@ def user_ldifs( strict=False, ) # loop through the records - for (dn,) in parser.parse(): + for dn, record in parser.parse(): log.info(f"Got entry record: {dn}") # for each user find child entries @@ -518,9 +514,7 @@ def main( f"{clone_path}/**/*", recursive=True, ) - if Path(file).is_file() - and Path(file).name.endswith(".ldif") - or Path(file).name.endswith(".j2") + if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") ] prep_for_templating(files) From 58759cc925f00486bd911213af3dd22e1de276c6 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Wed, 22 Nov 2023 16:23:39 +0000 Subject: [PATCH 27/29] migration to python-ldap - correction on tree deletion (#28) --- cli/__init__.py | 6 +- cli/ldap/rbac.py | 527 ---------------------------- cli/{ldap => ldap_cmds}/__init__.py | 0 cli/ldap_cmds/rbac.py | 518 +++++++++++++++++++++++++++ cli/{ldap => ldap_cmds}/user.py | 48 +-- requirements-dev.txt | 2 +- requirements.txt | 12 +- 7 files changed, 546 insertions(+), 567 deletions(-) delete mode 100644 cli/ldap/rbac.py rename cli/{ldap => ldap_cmds}/__init__.py (100%) create mode 100644 cli/ldap_cmds/rbac.py rename cli/{ldap => ldap_cmds}/user.py (91%) diff --git a/cli/__init__.py b/cli/__init__.py index 4ea2b0b..f370ca6 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,6 +1,6 @@ import click -import cli.ldap.rbac -import cli.ldap.user +import cli.ldap_cmds.rbac +import cli.ldap_cmds.user from cli import ( logger, @@ -166,7 +166,7 @@ def update_user_roles( def rbac_uplift( rbac_repo_tag, ): - cli.ldap.rbac.main(rbac_repo_tag) + cli.ldap_cmds.rbac.main(rbac_repo_tag) @click.command() diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py deleted file mode 100644 index c7b6204..0000000 --- a/cli/ldap/rbac.py +++ /dev/null @@ -1,527 +0,0 @@ -import ldap3.utils.hashed -from cli.ldap import ( - ldap_connect, -) -from cli import ( - env, -) -import cli.git as git -import glob -from cli.logger import ( - log, -) -from pathlib import ( - Path, -) -import cli.template -from ldif import ( - LDIFParser, -) - -# example for token auth -# def get_repo_with_token(repo_tag="master"): -# app_id = env.vars.get("GH_APP_ID") -# private_key = env.vars.get("GH_PRIVATE_KEY") -# installation_id = env.vars.get("GH_INSTALLATION_ID") -# token = git.get_access_token(app_id, private_key, installation_id) - - -ldap_config = { - "bind_user": "cn=root,dc=moj,dc=com", - "bind_user_cn": "root", - "base_root": "dc=moj,dc=com", - "base_root_dc": "moj", - "base_users": "ou=Users,dc=moj,dc=com", - "base_users_ou": "Users", - "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", - "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", - "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", - "base_groups": "ou=groups,dc=moj,dc=com", - "base_groups_ou": "groups", -} - - -def get_repo( - repo_tag="master", -): - url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" - try: - repo = git.get_repo( - url, - dest_name="rbac", - branch_or_tag=repo_tag, - ) - return repo - except Exception as e: - log.exception(e) - return None - - -def prep_for_templating( - files, - strings=None, -): - rbac_substitutions = { - "bind_password_hash.stdout": "bind_password_hash", - r"ldap_config.base_users | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_users_ou", - r"ldap_config.base_root | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_root_dc", - r"ldap_config.base_groups | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_groups_ou", - r"ldap_config.bind_user | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.bind_user_cn", - "'/'+environment_name+'/'+project_name+'": "", - "/gdpr/api/": "'gdpr_api_", - "/pwm/pwm/config_password": "'pwm_config_password", - "/merge/api/client_secret": "'merge_api_client_secret", - "/weblogic/ndelius-domain/umt_client_secret": "'umt_client_secret", - "ssm_prefix + ": "", - "cn=Users,dc=pcms,dc=internal": "ou=Users,dc=moj,dc=com", - "ssm_prefix+": "", - } - - if strings is None: - strings = env.vars.get("RBAC_SUBSTITUTIONS") or rbac_substitutions - - for file_path in files: - file = Path(file_path) - log.info("Replacing strings in rbac files") - for ( - k, - v, - ) in strings.items(): - log.debug(f"replacing {k} with {v} in {file_path}") - file.write_text( - file.read_text().replace( - k, - v, - ), - ) - - -def template_rbac( - files, -): - hashed_pwd_admin_user = ldap3.utils.hashed.hashed( - ldap3.HASHED_SALTED_SHA, - env.secrets.get("LDAP_ADMIN_PASSWORD"), - ) - rendered_files = [] - - for file in files: - rendered_text = cli.template.render( - file, - ldap_config=env.vars.get("LDAP_CONFIG") or ldap_config, - bind_password_hash=hashed_pwd_admin_user, - secrets=env.secrets, - oasys_password=env.secrets.get("OASYS_PASSWORD"), - environment_name=env.vars.get("ENVIRONMENT_NAME"), - project_name=env.vars.get("PROJECT_NAME"), - ) - rendered_file = cli.template.save( - rendered_text, - file, - ) - rendered_files.append(rendered_file) - return rendered_files - - -def context_ldif( - rendered_files, -): - context_file = [file for file in rendered_files if "context" in Path(file).name] - for file in context_file: - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - for ( - dn, - record, - ) in parser.parse(): - log.info(f"got entry record: {dn}") - log.debug(record) - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - - try: - ldap_connection.add( - dn, - attributes=record, - ) - log.debug(ldap_connection.result["result"]) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info("Successfully added context") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -def group_ldifs( - rendered_files, -): - # connect to ldap - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - - group_files = [file for file in rendered_files if "groups" in Path(file).name] - # loop through the group files - for file in group_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for ( - dn, - record, - ) in parser.parse(): - log.debug(f"got entry record: {dn}") - log.debug(record) - # add the record to ldap - try: - ldap_connection.add( - dn, - attributes=record, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if record.get("description"): - log.info(f"Updating description for {record}") - try: - ldap_connection.modify( - dn, - { - "description": [ - ( - ldap3.MODIFY_REPLACE, - record["description"], - ) - ] - }, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info(f"Successfully added groups") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -def policy_ldifs( - rendered_files, -): - # connect to ldap - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - - policy_files = [file for file in rendered_files if "policy" in Path(file).name] - - # first, delete the policies - ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config - ldap_connection.delete("ou=Policies," + ldap_config_dict.get("base_root")) - - # loop through the policy files - for file in policy_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for ( - dn, - record, - ) in parser.parse(): - log.info(f"Got entry record: {dn}") - # add the record to ldap - try: - ldap_connection.add( - dn, - attributes=record, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info(f"Successfully added policies") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -def role_ldifs( - rendered_files, -): - # connect to ldap - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - role_files = [file for file in rendered_files if "nd_role" in Path(file).name] - - # first, delete the roles - ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config - ldap_connection.delete("cn=ndRoleCatalogue," + ldap_config_dict.get("base_users")) - ldap_connection.delete("cn=ndRoleGroups," + ldap_config_dict.get("base_users")) - - # ensure boolean values are Uppercase.. this comes from the ansible yml - # (not yet implemented, probably not needed) - - # loop through the role files - for file in role_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for ( - dn, - record, - ) in parser.parse(): - log.info(f"Got entry record: {dn}") - # add the record to ldap - try: - ldap_connection.add( - dn, - attributes=record, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info(f"Successfully added roles") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -# not complete!! -# see https://github.com/ministryofjustice/hmpps-delius-pipelines/blob/master/components/delius-core/playbooks/rbac/import_schemas.yml -def schema_ldifs( - rendered_files, -): - # connect to ldap - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - - schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] - - # loop through the schema files - for file in schema_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for ( - dn, - record, - ) in parser.parse(): - log.info(f"Got entry record: {dn}") - # add the record to ldap - try: - ldap_connection.add( - dn, - attributes=record, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info(f"Successfully added schemas") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -def user_ldifs( - rendered_files, -): - # connect to ldap - try: - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), - ) - except Exception as e: - log.exception(f"Failed to connect to ldap") - raise e - - user_files = [file for file in rendered_files if "-users" in Path(file).name] - - # first, delete the users - for file in user_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for dn, record in parser.parse(): - log.info(f"Got entry record: {dn}") - - # for each user find child entries - try: - ldap_connection.search( - dn, - "(objectclass=*)", - search_scope=ldap3.SUBTREE, - ) - except Exception as e: - log.exception(f"Failed to search {dn}") - raise e - - # delete child entries - try: - for entry in ldap_connection.entries: - log.debug(entry.entry_dn) - ldap_connection.delete(entry.entry_dn) - except Exception as e: - log.exception(f"Failed to delete {entry.entry_dn}") - raise e - - try: - ldap_connection.delete(dn) - except Exception as e: - log.exception(f"Failed to delete {dn}") - raise e - - # loop through the user files - for file in user_files: - # parse the ldif into dn and record - parser = LDIFParser( - open( - file, - "rb", - ), - strict=False, - ) - # loop through the records - for ( - dn, - record, - ) in parser.parse(): - log.info(f"Got entry record: {dn}") - - # add the record to ldap - try: - ldap_connection.add( - dn, - attributes=record, - ) - except Exception as e: - log.exception(f"Failed to add {dn}... {record}") - raise e - - if ldap_connection.result["result"] == 0: - log.info(f"Successfully added users") - elif ldap_connection.result["result"] == 68: - log.info(f"{dn} already exists") - else: - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add {dn}... {record}") - - -def main( - rbac_repo_tag, - clone_path="./rbac", -): - get_repo(rbac_repo_tag) - files = [ - file - for file in glob.glob( - f"{clone_path}/**/*", - recursive=True, - ) - if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") - ] - - prep_for_templating(files) - rendered_files = template_rbac(files) - context_ldif(rendered_files) - policy_ldifs(rendered_files) - # schema_ldifs(files) probably not needed, but need to check! - role_ldifs(rendered_files) - group_ldifs(rendered_files) - user_ldifs(rendered_files) diff --git a/cli/ldap/__init__.py b/cli/ldap_cmds/__init__.py similarity index 100% rename from cli/ldap/__init__.py rename to cli/ldap_cmds/__init__.py diff --git a/cli/ldap_cmds/rbac.py b/cli/ldap_cmds/rbac.py new file mode 100644 index 0000000..33a30b4 --- /dev/null +++ b/cli/ldap_cmds/rbac.py @@ -0,0 +1,518 @@ +from pprint import pprint + +import ldap +import ldap3.utils.hashed +import ldif +import ldap.modlist as modlist + +from cli.ldap_cmds import ( + ldap_connect, +) +from cli import ( + env, +) +import cli.git as git +import glob +from cli.logger import ( + log, +) +from pathlib import ( + Path, +) +import cli.template + +# example for token auth +# def get_repo_with_token(repo_tag="master"): +# app_id = env.vars.get("GH_APP_ID") +# private_key = env.vars.get("GH_PRIVATE_KEY") +# installation_id = env.vars.get("GH_INSTALLATION_ID") +# token = git.get_access_token(app_id, private_key, installation_id) + + +ldap_config = { + "bind_user": "cn=root,dc=moj,dc=com", + "bind_user_cn": "root", + "base_root": "dc=moj,dc=com", + "base_root_dc": "moj", + "base_users": "ou=Users,dc=moj,dc=com", + "base_users_ou": "Users", + "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", + "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", + "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", + "base_groups": "ou=groups,dc=moj,dc=com", + "base_groups_ou": "groups", +} + + +def get_repo( + repo_tag="master", +): + url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" + try: + repo = git.get_repo( + url, + dest_name="rbac", + branch_or_tag=repo_tag, + ) + return repo + except Exception as e: + log.exception(e) + return None + + +def prep_for_templating( + files, + strings=None, +): + rbac_substitutions = { + "bind_password_hash.stdout": "bind_password_hash", + r"ldap_config.base_users | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_users_ou", + r"ldap_config.base_root | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_root_dc", + r"ldap_config.base_groups | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_groups_ou", + r"ldap_config.bind_user | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.bind_user_cn", + "'/'+environment_name+'/'+project_name+'": "", + "/gdpr/api/": "'gdpr_api_", + "/pwm/pwm/config_password": "'pwm_config_password", + "/merge/api/client_secret": "'merge_api_client_secret", + "/weblogic/ndelius-domain/umt_client_secret": "'umt_client_secret", + "ssm_prefix + ": "", + "cn=Users,dc=pcms,dc=internal": "ou=Users,dc=moj,dc=com", + "ssm_prefix+": "", + } + + if strings is None: + strings = env.vars.get("RBAC_SUBSTITUTIONS") or rbac_substitutions + + for file_path in files: + file = Path(file_path) + log.info("Replacing strings in rbac files") + for ( + k, + v, + ) in strings.items(): + log.debug(f"replacing {k} with {v} in {file_path}") + file.write_text( + file.read_text().replace( + k, + v, + ), + ) + + +def template_rbac( + files, +): + hashed_pwd_admin_user = ldap3.utils.hashed.hashed( + ldap3.HASHED_SALTED_SHA, + env.secrets.get("LDAP_ADMIN_PASSWORD"), + ) + rendered_files = [] + + for file in files: + rendered_text = cli.template.render( + file, + ldap_config=env.vars.get("LDAP_CONFIG") or ldap_config, + bind_password_hash=hashed_pwd_admin_user, + secrets=env.secrets, + oasys_password=env.secrets.get("OASYS_PASSWORD"), + environment_name=env.vars.get("ENVIRONMENT_NAME"), + project_name=env.vars.get("PROJECT_NAME"), + ) + rendered_file = cli.template.save( + rendered_text, + file, + ) + rendered_files.append(rendered_file) + return rendered_files + + +def context_ldif( + rendered_files, +): + context_file = [file for file in rendered_files if "context" in Path(file).name] + + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + for file in context_file: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"got entry record: {dn}") + log.debug(attributes) + + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def group_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + group_files = [file for file in rendered_files if "-groups" in Path(file).name] + # loop through the group files + for file in group_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.debug(f"got entry record: {dn}") + log.debug(attributes) + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + if attributes.get("description"): + log.info(f"Updating description for {dn}") + try: + connection.modify(dn, [(ldap.MOD_REPLACE, "description", attributes["description"])]) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def policy_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + policy_files = [file for file in rendered_files if "policy" in Path(file).name] + + # first, delete the policies + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + policy_tree = "ou=Policies," + ldap_config_dict.get("base_root") + + tree = connection.search_s( + policy_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + + # loop through the policy files + for file in policy_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def role_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + role_files = [file for file in rendered_files if "nd_role" in Path(file).name] + + # first, delete the roles + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + + role_trees = [ + "cn=ndRoleCatalogue," + ldap_config_dict.get("base_users"), + "cn=ndRoleGroups," + ldap_config_dict.get("base_users"), + ] + for role_tree in role_trees: + tree = connection.search_s( + role_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + + # ensure boolean values are Uppercase.. this comes from the ansible yml + # (not yet implemented, probably not needed) + + # loop through the role files + for file in role_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +# not complete!! +# see https://github.com/ministryofjustice/hmpps-delius-pipelines/blob/master/components/delius-core/playbooks/rbac/import_schemas.yml +def schema_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] + + # loop through the schema files + for file in schema_files: + # parse the ldif into dn and record + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + # loop through the records + for entry in records.all_records: + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + dn = entry[0] + attributes = entry[1] + print(f" {entry[0]}") + connection.add_s(dn, modlist.addModlist(attributes)) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def user_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + user_files = [file for file in rendered_files if "-users.ldif" in Path(file).name] + + # first, delete the users + for file in user_files: + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + for record in records.all_records: + dn = record[0] + log.info(f"Got entry record: {dn}") + try: + # search for dn children + tree = connection.search_s( + dn, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + print(tree) + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + # connection.delete_ext_s(dn, serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + # print(f"Deleted {dn}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + except Exception as e: + log.exception(e) + raise e + + for file in user_files: + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + print(f" {entry[0]}") + connection.add_s(dn, modlist.addModlist(attributes)) + + # connect to ldap + # try: + # ldap_connection_addition = ldap_connect( + # env.vars.get("LDAP_HOST"), + # env.vars.get("LDAP_USER"), + # env.secrets.get("LDAP_BIND_PASSWORD"), + # ) + # except Exception as e: + # log.exception(f"Failed to connect to ldap") + # raise e + + # loop through the user files + # for file in user_files: + # # parse the ldif into dn and record + # parser = LDIFParser( + # open( + # file, + # "rb", + # ), + # strict=False, + # ) + # # loop through the records + # for ( + # dn, + # record, + # ) in parser.parse(): + # log.info(f"Got entry record: {dn}") + # + # # add the record to ldap + # try: + # print(dn) + # print(record) + # ldap_connection_addition.add( + # dn, + # record, + # ) + # except Exception as e: + # log.exception(f"Failed to add {dn}... {record}") + # raise e + # + # if ldap_connection_addition.result["result"] == 0: + # log.info(f"Successfully added users") + # elif ldap_connection_addition.result["result"] == 68: + # log.info(f"{dn} already exists") + # else: + # log.debug(ldap_connection_addition.result) + # log.debug(ldap_connection_addition.response) + # raise Exception(f"Failed to add {dn}... {record}") + + +def main( + rbac_repo_tag, + clone_path="./rbac", +): + get_repo(rbac_repo_tag) + files = [ + file + for file in glob.glob( + f"{clone_path}/**/*", + recursive=True, + ) + if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") + ] + + prep_for_templating(files) + rendered_files = template_rbac(files) + context_ldif(rendered_files) + policy_ldifs(rendered_files) + # schema_ldifs(files) probably not needed, but need to check! + role_ldifs(rendered_files) + group_ldifs(rendered_files) + user_ldifs(rendered_files) diff --git a/cli/ldap/user.py b/cli/ldap_cmds/user.py similarity index 91% rename from cli/ldap/user.py rename to cli/ldap_cmds/user.py index 1aa4efa..410a801 100644 --- a/cli/ldap/user.py +++ b/cli/ldap_cmds/user.py @@ -1,6 +1,6 @@ import oracledb -import cli.ldap +import cli.ldap_cmds from cli.logger import ( log, @@ -9,7 +9,7 @@ env, ) -from cli.ldap import ( +from cli.ldap_cmds import ( ldap_connect, ) from ldap3 import ( @@ -45,7 +45,9 @@ def change_home_areas( env.secrets.get("LDAP_BIND_PASSWORD"), ) - search_filter = f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + search_filter = ( + f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + ) ldap_connection.search( ",".join( [ @@ -77,9 +79,7 @@ def change_home_areas( if ldap_connection.result["result"] == 0: log.info(f"Successfully updated {attribute} for {dn}") else: - log.error( - f"Failed to update {attribute} for {dn}: {ldap_connection.result}" - ) + log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") ######################################### @@ -95,10 +95,7 @@ def parse_user_role_list( # and the roles are separated by a semi-colon: # username1,role1;role2;role3|username2,role1;role2 - return { - user.split(",")[0]: user.split(",")[1].split(";") - for user in user_role_list.split("|") - } + return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} def add_roles_to_user( @@ -208,13 +205,7 @@ def update_roles( log.exception("Failed to search for users") raise e - users_found = sorted( - [ - entry.cn.value - for entry in ldap_connection_user_filter.entries - if entry.cn.value - ] - ) + users_found = sorted([entry.cn.value for entry in ldap_connection_user_filter.entries if entry.cn.value]) log.debug("users found from user filter") log.debug(users_found) ldap_connection_user_filter.unbind() @@ -224,7 +215,9 @@ def update_roles( # create role filter if len(roles_filter_list) > 0: - full_role_filter = f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" + full_role_filter = ( + f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" + ) else: full_role_filter = "(&(objectclass=NDRoleAssociation)(cn=*))" @@ -257,12 +250,7 @@ def update_roles( raise e roles_found = sorted( - set( - { - entry.entry_dn.split(",")[1].split("=")[1] - for entry in ldap_connection_role_filter.entries - } - ) + set({entry.entry_dn.split(",")[1].split("=")[1] for entry in ldap_connection_role_filter.entries}) ) log.debug("users found from roles filter: ") log.debug(roles_found) @@ -321,9 +309,7 @@ def update_roles( log.e(f"Failed to add role '{item[1]}' to user '{item[0]}'") log.debug(ldap_connection_action.result) elif remove: - ldap_connection_action.delete( - f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}" - ) + ldap_connection_action.delete(f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}") if ldap_connection_action.result["result"] == 0: log.info(f"Successfully removed role '{item[1]}' from user '{item[0]}'") elif ldap_connection_action.result["result"] == 32: @@ -402,9 +388,7 @@ def deactivate_crc_users( env.secrets.get("LDAP_BIND_PASSWORD"), ) - user_filter = ( - "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" - ) + user_filter = "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" home_areas = [ [ @@ -479,7 +463,9 @@ def deactivate_crc_users( connection = cli.database.connection() for user_dn in all_users: try: - update_sql = f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + update_sql = ( + f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + ) update_cursor = connection.cursor() update_cursor.execute( update_sql, diff --git a/requirements-dev.txt b/requirements-dev.txt index ae204bd..77bfeb0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,8 +6,8 @@ PyGithub GitPython pyjwt python-dotenv +python-ldap Jinja2 -git+https://github.com/abilian/ldif.git@4.2.0 pre-commit black flake8 diff --git a/requirements.txt b/requirements.txt index 79a4e4b..f2550d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ click==8.1.6 -ldap3 +ldap3~=2.9.1 oracledb==1.4 ansible-runner PyGithub GitPython==3.1.37 -pyjwt -python-dotenv -Jinja2 -git+https://github.com/abilian/ldif.git@4.2.0 \ No newline at end of file +pyjwt~=2.8.0 +python-dotenv~=1.0.0 +Jinja2~=3.1.2 +python-ldap +requests~=2.31.0 +setuptools~=68.2.2 \ No newline at end of file From b7188c96e609675aef32e419a0b8cd98d382b09e Mon Sep 17 00:00:00 2001 From: George Taylor Date: Thu, 23 Nov 2023 10:10:32 +0000 Subject: [PATCH 28/29] Merge branch 'main' into dev --- cli/ansible/__init__.py | 2 -- cli/config.py | 17 ----------------- 2 files changed, 19 deletions(-) delete mode 100644 cli/ansible/__init__.py delete mode 100644 cli/config.py diff --git a/cli/ansible/__init__.py b/cli/ansible/__init__.py deleted file mode 100644 index 98c7787..0000000 --- a/cli/ansible/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import ansible_runner - diff --git a/cli/config.py b/cli/config.py deleted file mode 100644 index 5a7343a..0000000 --- a/cli/config.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from dotenv import load_dotenv - -load_dotenv() - -ldap_host = os.getenv("LDAP_HOST") -ldap_user = os.getenv("LDAP_USER") -ldap_password = os.getenv("LDAP_PASSWORD") -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_service_name = os.getenv("DB_SERVICE_NAME") -gh_app_id = os.getenv("GH_APP_ID") -gh_private_key = os.getenv("GH_PRIVATE_KEY") -gh_installation_id = os.getenv("GH_INSTALLATION_ID") \ No newline at end of file From 604a5a942c316937bbfde7a782acacb2a214a980 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Thu, 23 Nov 2023 10:11:41 +0000 Subject: [PATCH 29/29] Update .flake8 --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 7ca5edc..5c8f544 100644 --- a/.flake8 +++ b/.flake8 @@ -31,4 +31,4 @@ extend-ignore = # flake-quote settings: # ===================== # Set this to match black style: -inline-quotes = double \ No newline at end of file +inline-quotes = double