diff --git a/.github/scripts/update_data.py b/.github/scripts/update_data.py index a2e4efcb3..f3dd5334d 100644 --- a/.github/scripts/update_data.py +++ b/.github/scripts/update_data.py @@ -7,5 +7,5 @@ from policy_sentry.command.initialize import initialize -if __name__ == '__main__': +if __name__ == "__main__": initialize(None, True, True) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c4766f19..5e87d9f45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/rhysd/actionlint - rev: v1.6.26 + rev: v1.7.1 hooks: - id: actionlint-docker - repo: https://github.com/antonbabenko/pre-commit-terraform @@ -10,12 +10,9 @@ repos: # - id: terraform_docs # args: ['--sort-by-required', '--no-providers'] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.6 + rev: v0.5.1 hooks: + - id: ruff + files: ^policy_sentry/ - id: ruff-format files: ^policy_sentry/ - - repo: https://github.com/asottile/pyupgrade - rev: v3.11.0 - hooks: - - id: pyupgrade - args: ["--py37-plus"] diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index d41375f23..000000000 --- a/.pylintrc +++ /dev/null @@ -1,525 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - fixme, - line-too-long, - duplicate-code, - consider-using-sys-exit, - no-else-return, - too-few-public-methods, - too-many-nested-blocks, - too-many-statements, - too-many-branches, - self-assigning-variable, - pointless-string-statement, - too-many-locals, - consider-using-enumerate, - too-many-arguments, - expression-not-assigned, - invalid-name, - logging-too-many-args, - logging-format-interpolation, - f-string-without-interpolation, - logging-fstring-interpolation, - unused-variable, - broad-exception-raised, - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=119 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[STRING] - -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. -check-str-concat-over-line-jumps=no - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=builtins.BaseException, - builtins.Exception diff --git a/Makefile b/Makefile index c38edfbf9..6cc7a87cf 100644 --- a/Makefile +++ b/Makefile @@ -50,15 +50,6 @@ clean: # --------------------------------------------------------------------------------------------------------------------- test: setup-dev python3 -m coverage run -m pytest -v -security-test: setup-dev - bandit -r ./${PROJECT_UNDERSCORE}/ -# --------------------------------------------------------------------------------------------------------------------- -# Linting and formatting -# --------------------------------------------------------------------------------------------------------------------- -fmt: setup-dev - black ${PROJECT_UNDERSCORE}/ -lint: setup-dev - pylint ${PROJECT_UNDERSCORE}/ # --------------------------------------------------------------------------------------------------------------------- # Package publishing # --------------------------------------------------------------------------------------------------------------------- diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index f13158119..ddc0d8cef 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -48,8 +48,7 @@ Available tasks: integration.version Print the version integration.write-policy Integration testing: Tests the `write-policy` function. - test.lint Linting with `pylint` and `autopep8` - test.security Runs `bandit` and `safety check` + test.security Runs `safety check` unit.pytest Unit testing: Runs unit tests using `pytest` diff --git a/policy_sentry/analysis/expand.py b/policy_sentry/analysis/expand.py index 317806184..86d5cb8e7 100644 --- a/policy_sentry/analysis/expand.py +++ b/policy_sentry/analysis/expand.py @@ -2,9 +2,9 @@ from __future__ import annotations -import logging import copy import fnmatch +import logging from typing import Any from policy_sentry.querying.actions import get_actions_for_service diff --git a/policy_sentry/bin/cli.py b/policy_sentry/bin/cli.py index b9add3b05..aa7f46085 100755 --- a/policy_sentry/bin/cli.py +++ b/policy_sentry/bin/cli.py @@ -4,6 +4,7 @@ """ import click + from policy_sentry import command from policy_sentry.bin.version import __version__ diff --git a/policy_sentry/command/__init__.py b/policy_sentry/command/__init__.py index b6ca93147..8f200ae79 100644 --- a/policy_sentry/command/__init__.py +++ b/policy_sentry/command/__init__.py @@ -1,5 +1,2 @@ -# pylint: disable=missing-module-docstring -from policy_sentry.command import initialize -from policy_sentry.command import write_policy -from policy_sentry.command import create_template -from policy_sentry.command import query +# ruff: noqa: F401 +from policy_sentry.command import create_template, initialize, query, write_policy diff --git a/policy_sentry/command/create_template.py b/policy_sentry/command/create_template.py index 73bb6083c..d145c05ad 100644 --- a/policy_sentry/command/create_template.py +++ b/policy_sentry/command/create_template.py @@ -5,11 +5,13 @@ from __future__ import annotations -from pathlib import Path import logging +from pathlib import Path + import click -from policy_sentry.writing.template import create_actions_template, create_crud_template + from policy_sentry import set_stream_logger +from policy_sentry.writing.template import create_actions_template, create_crud_template logger = logging.getLogger(__name__) diff --git a/policy_sentry/command/initialize.py b/policy_sentry/command/initialize.py index 00729f030..161ad2433 100755 --- a/policy_sentry/command/initialize.py +++ b/policy_sentry/command/initialize.py @@ -5,26 +5,28 @@ from __future__ import annotations +import logging import os import shutil -import logging + import click + +from policy_sentry import set_stream_logger from policy_sentry.querying.all import get_all_service_prefixes from policy_sentry.shared.awsdocs import ( - update_html_docs_directory, create_database, + update_html_docs_directory, ) from policy_sentry.shared.constants import ( - LOCAL_HTML_DIRECTORY_PATH, + BUNDLED_DATA_DIRECTORY, + BUNDLED_DATASTORE_FILE_PATH, + BUNDLED_HTML_DIRECTORY_PATH, CONFIG_DIRECTORY, - LOCAL_DATASTORE_FILE_PATH, DATASTORE_FILE_PATH, LOCAL_ACCESS_OVERRIDES_FILE, - BUNDLED_HTML_DIRECTORY_PATH, - BUNDLED_DATASTORE_FILE_PATH, - BUNDLED_DATA_DIRECTORY, + LOCAL_DATASTORE_FILE_PATH, + LOCAL_HTML_DIRECTORY_PATH, ) -from policy_sentry import set_stream_logger logger = logging.getLogger(__name__) diff --git a/policy_sentry/command/query.py b/policy_sentry/command/query.py index 76a7f0928..521dc8a4d 100644 --- a/policy_sentry/command/query.py +++ b/policy_sentry/command/query.py @@ -4,51 +4,51 @@ from __future__ import annotations -import os import json import logging +import os from typing import Any import click import yaml -from policy_sentry.querying.services import get_services_data -from policy_sentry.util.access_levels import transform_access_level_text +from policy_sentry import set_stream_logger +from policy_sentry.querying.actions import ( + get_action_data, + get_actions_for_service, + get_actions_matching_arn_type, + get_actions_matching_condition_key, + get_actions_that_support_wildcard_arns_only, + get_actions_with_access_level, + get_actions_with_arn_type_and_access_level, +) from policy_sentry.querying.all import get_all_service_prefixes from policy_sentry.querying.arns import ( get_arn_type_details, get_arn_types_for_service, get_raw_arns_for_service, ) -from policy_sentry.querying.actions import ( - get_actions_for_service, - get_actions_with_access_level, - get_action_data, - get_actions_matching_condition_key, - get_actions_with_arn_type_and_access_level, - get_actions_matching_arn_type, - get_actions_that_support_wildcard_arns_only, -) from policy_sentry.querying.conditions import ( - get_condition_keys_for_service, get_condition_key_details, + get_condition_keys_for_service, ) +from policy_sentry.querying.services import get_services_data from policy_sentry.shared.constants import ( DATASTORE_FILE_PATH, LOCAL_DATASTORE_FILE_PATH, ) -from policy_sentry import set_stream_logger +from policy_sentry.util.access_levels import transform_access_level_text logger = logging.getLogger(__name__) iam_definition_path = DATASTORE_FILE_PATH -def print_list(output: Any, fmt: str = "json") -> None: +def print_list(output: list[Any], fmt: str = "json") -> None: """Common method on how to print a list, depending on whether the user requests JSON or YAML output""" print(yaml.dump(output)) if fmt == "yaml" else [print(item) for item in output] -def print_dict(output: Any, fmt: str = "json") -> None: +def print_dict(output: list[Any] | dict[Any, Any], fmt: str = "json") -> None: """Common method on how to print a dict, depending on whether the user requests JSON, YAML or CSV output""" if fmt == "csv": if not output: diff --git a/policy_sentry/command/write_policy.py b/policy_sentry/command/write_policy.py index 7ff27bbde..a1fc0f00e 100755 --- a/policy_sentry/command/write_policy.py +++ b/policy_sentry/command/write_policy.py @@ -4,9 +4,9 @@ from __future__ import annotations -import sys import json import logging +import sys from pathlib import Path from typing import Any @@ -14,9 +14,9 @@ import yaml from click import Context +from policy_sentry import set_stream_logger from policy_sentry.util.file import read_yaml_file from policy_sentry.writing.sid_group import SidGroup -from policy_sentry import set_stream_logger logger = logging.getLogger(__name__) @@ -47,7 +47,7 @@ class RegisterMinimizeLengthCommand(click.Command): def parse_args(self, ctx: Context, args: list[str]) -> list[str]: options = [o for o in ctx.command.params if getattr(o, "register_length", None)] - prefixes = {p for p in sum([o.opts for o in options], []) if p.startswith("--")} + prefixes = {p for p in sum([o.opts for o in options], []) if p.startswith("--")} # noqa: RUF017 for i, a in enumerate(args): a_tuple = a.split("=") if a_tuple[0] in prefixes: diff --git a/policy_sentry/querying/actions.py b/policy_sentry/querying/actions.py index eddacb669..6da801737 100644 --- a/policy_sentry/querying/actions.py +++ b/policy_sentry/querying/actions.py @@ -5,29 +5,29 @@ from __future__ import annotations -import logging import functools +import logging from typing import Any from policy_sentry.querying.actions_v1 import ( get_action_data_v1, get_action_matching_access_level_v1, - get_actions_with_arn_type_and_access_level_v1, + get_actions_for_service_v1, get_actions_matching_arn_type_v1, get_actions_matching_arn_v1, - get_actions_for_service_v1, -) -from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2 -from policy_sentry.shared.iam_data import ( - iam_definition, - get_service_prefix_data, - get_iam_definition_schema_version, + get_actions_with_arn_type_and_access_level_v1, ) -from policy_sentry.querying.all import get_all_service_prefixes, get_all_actions +from policy_sentry.querying.all import get_all_actions, get_all_service_prefixes from policy_sentry.querying.arns import ( get_matching_raw_arns, get_resource_type_name_with_raw_arn, ) +from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2 +from policy_sentry.shared.iam_data import ( + get_iam_definition_schema_version, + get_service_prefix_data, + iam_definition, +) from policy_sentry.util.arns import get_service_from_arn all_service_prefixes = get_all_service_prefixes() @@ -320,12 +320,12 @@ def get_actions_with_arn_type_and_access_level_v2( else: service_prefix_data = get_service_prefix_data(service_prefix) for action_name, action_data in service_prefix_data["privileges"].items(): - if action_data["access_level"] == access_level: - if ( - resource_type_name.lower() - in action_data["resource_types_lower_name"] - ): - results.append(f"{service_prefix}:{action_name}") + if ( + action_data["access_level"] == access_level + and resource_type_name.lower() + in action_data["resource_types_lower_name"] + ): + results.append(f"{service_prefix}:{action_name}") return results @@ -352,9 +352,10 @@ def get_actions_that_support_wildcard_arns_only(service_prefix: str) -> list[str else: service_prefix_data = get_service_prefix_data(service_prefix) for action_name, action_data in service_prefix_data["privileges"].items(): - if len(action_data["resource_types"]) == 1: - if action_data["resource_types"].get(""): - results.append(f"{service_prefix}:{action_name}") + if len(action_data["resource_types"]) == 1 and action_data[ + "resource_types" + ].get(""): + results.append(f"{service_prefix}:{action_name}") return results @@ -469,8 +470,8 @@ def get_actions_matching_condition_key( Returns: List: A list of actions """ - results = [] if service_prefix == "all": + results = [] for some_prefix in all_service_prefixes: actions = get_actions_matching_condition_key( service_prefix=some_prefix, @@ -480,10 +481,12 @@ def get_actions_matching_condition_key( results.extend(actions) else: service_prefix_data = get_service_prefix_data(service_prefix) - for action_name, action_data in service_prefix_data["privileges"].items(): - for resource_data in action_data["resource_types"].values(): - if condition_key in resource_data["condition_keys"]: - results.append(f"{service_prefix}:{action_name}") + results = [ + f"{service_prefix}:{action_name}" + for action_name, action_data in service_prefix_data["privileges"].items() + for resource_data in action_data["resource_types"].values() + if condition_key in resource_data["condition_keys"] + ] return results @@ -637,10 +640,12 @@ def remove_actions_that_are_not_wildcard_arn_only(actions_list: list[str]) -> li for action in actions_list_unique: service_prefix, action_name = action.split(":") action_data = get_action_data(service_prefix, action_name) - if len(action_data[service_prefix]) == 1: - if action_data[service_prefix][0]["resource_arn_format"] == "*": - # Let's return the CamelCase action name format - results.append(action_data[service_prefix][0]["action"]) + if ( + len(action_data[service_prefix]) == 1 + and action_data[service_prefix][0]["resource_arn_format"] == "*" + ): + # Let's return the CamelCase action name format + results.append(action_data[service_prefix][0]["action"]) return results diff --git a/policy_sentry/querying/actions_v1.py b/policy_sentry/querying/actions_v1.py index 691ecf8aa..b8ca3f229 100644 --- a/policy_sentry/querying/actions_v1.py +++ b/policy_sentry/querying/actions_v1.py @@ -29,7 +29,9 @@ def get_actions_for_service_v1( Returns: List: A list of actions """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) service_prefix_data = get_service_prefix_data(service_prefix) results = [] @@ -62,7 +64,9 @@ def get_action_data_v1( Returns: List: A dictionary containing metadata about an IAM Action. """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) results = [] action_data_results = {} @@ -74,17 +78,17 @@ def get_action_data_v1( # Get the baseline conditions and dependent actions condition_keys = [] dependent_actions = [] - rows = [] if action_name == "*": - # rows = this_action_data["resource_types"] - for resource_type_entry in this_action_data["resource_types"]: - rows.append(this_action_data["resource_types"][resource_type_entry]) + rows = [ + this_action_data["resource_types"][resource_type_entry] + for resource_type_entry in this_action_data["resource_types"] + ] else: - for resource_type_entry in this_action_data["resource_types"]: - if this_action_name.lower() == action_name.lower(): - rows.append( - this_action_data["resource_types"][resource_type_entry] - ) + rows = [ + this_action_data["resource_types"][resource_type_entry] + for resource_type_entry in this_action_data["resource_types"] + if this_action_name.lower() == action_name.lower() + ] for row in rows: # Set default value for if no other matches are found resource_arn_format = "*" @@ -92,9 +96,7 @@ def get_action_data_v1( if row["dependent_actions"]: dependent_actions.extend(row["dependent_actions"]) # Get the condition keys - for service_resource_name, service_resource_data in service_prefix_data[ - "resources" - ].items(): + for service_resource_data in service_prefix_data["resources"].values(): if row["resource_type"] == "": continue if ( @@ -138,7 +140,9 @@ def get_action_matching_access_level_v1( Returns: List: action or None """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) result = None service_prefix_data = get_service_prefix_data(service_prefix.lower()) @@ -146,10 +150,12 @@ def get_action_matching_access_level_v1( privileges = service_prefix_data.get("privileges") if privileges: for this_action_name, action_data in privileges.items(): - if action_data["access_level"] == access_level: - if this_action_name.lower() == action_name.lower(): - result = f"{service_prefix}:{this_action_name}" - break + if ( + action_data["access_level"] == access_level + and this_action_name.lower() == action_name.lower() + ): + result = f"{service_prefix}:{this_action_name}" + break return result @@ -169,7 +175,9 @@ def get_actions_with_arn_type_and_access_level_v1( Return: List: A list of actions that have that ARN type and Access level """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) service_prefix_data = get_service_prefix_data(service_prefix) results = [] @@ -177,11 +185,9 @@ def get_actions_with_arn_type_and_access_level_v1( if service_prefix == "all": for some_prefix in all_service_prefixes: service_prefix_data = get_service_prefix_data(some_prefix) - for action_name, action_data in service_prefix_data["privileges"].items(): + for action_data in service_prefix_data["privileges"].values(): if action_data["access_level"] == access_level: - for resource_name, resource_data in action_data[ - "resource_types" - ].items(): + for resource_data in action_data["resource_types"].values(): this_resource_type = resource_data["resource_type"].strip("*") if this_resource_type.lower() == resource_type_name.lower(): results.append( @@ -189,11 +195,9 @@ def get_actions_with_arn_type_and_access_level_v1( ) break else: - for action_name, action_data in service_prefix_data["privileges"].items(): + for action_data in service_prefix_data["privileges"].values(): if action_data["access_level"] == access_level: - for resource_name, resource_data in action_data[ - "resource_types" - ].items(): + for resource_data in action_data["resource_types"].values(): this_resource_type = resource_data["resource_type"].strip("*") if this_resource_type.lower() == resource_type_name.lower(): results.append(f"{service_prefix}:{action_data['privilege']}") @@ -215,7 +219,9 @@ def get_actions_matching_arn_type_v1( Return: List: A list of actions that have that ARN type """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) service_prefix_data = get_service_prefix_data(service_prefix) results = [] @@ -223,17 +229,15 @@ def get_actions_matching_arn_type_v1( if service_prefix == "all": for some_prefix in all_service_prefixes: service_prefix_data = get_service_prefix_data(some_prefix) - for action_name, action_data in service_prefix_data["privileges"].items(): - for resource_name, resource_data in action_data[ - "resource_types" - ].items(): + for action_data in service_prefix_data["privileges"].values(): + for resource_data in action_data["resource_types"].values(): this_resource_type = resource_data["resource_type"].strip("*") if this_resource_type.lower() == resource_type_name.lower(): results.append(f"{service_prefix}:{action_data['privilege']}") break else: - for action_name, action_data in service_prefix_data["privileges"].items(): - for resource_name, resource_data in action_data["resource_types"].items(): + for action_data in service_prefix_data["privileges"].values(): + for resource_data in action_data["resource_types"].values(): this_resource_type = resource_data["resource_type"].strip("*") if this_resource_type.lower() == resource_type_name.lower(): results.append(f"{service_prefix}:{action_data['privilege']}") @@ -252,7 +256,9 @@ def get_actions_matching_arn_v1(arn: str) -> list[str]: Returns: List: A list of all actions that can match it. """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) raw_arns = get_matching_raw_arns(arn) results = [] @@ -263,9 +269,9 @@ def get_actions_matching_arn_v1(arn: str) -> list[str]: service_prefix = get_service_from_arn(raw_arn) service_prefix_data = get_service_prefix_data(service_prefix) - for action_name, action_data in service_prefix_data["privileges"].items(): + for action_data in service_prefix_data["privileges"].values(): # for some_action in service_prefix_data["privileges"]: - for resource_name, resource_data in action_data["resource_types"].items(): + for resource_data in action_data["resource_types"].values(): this_resource_type = resource_data["resource_type"].strip("*") if this_resource_type.lower() == resource_type_name.lower(): results.append(f"{service_prefix}:{action_data['privilege']}") diff --git a/policy_sentry/querying/all.py b/policy_sentry/querying/all.py index 72d6fc7ee..942419c5c 100644 --- a/policy_sentry/querying/all.py +++ b/policy_sentry/querying/all.py @@ -2,8 +2,8 @@ from __future__ import annotations -import logging import functools +import logging from policy_sentry.querying.all_v1 import get_all_actions_v1 from policy_sentry.shared.constants import ( @@ -11,9 +11,9 @@ POLICY_SENTRY_SCHEMA_VERSION_V2, ) from policy_sentry.shared.iam_data import ( - iam_definition, - get_service_prefix_data, get_iam_definition_schema_version, + get_service_prefix_data, + iam_definition, ) logger = logging.getLogger(__name__) @@ -31,8 +31,7 @@ def get_all_service_prefixes() -> set[str]: List: A list of all AWS service prefixes present in the table. """ results = set(iam_definition.keys()) - if POLICY_SENTRY_SCHEMA_VERSION_NAME in results: - results.remove(POLICY_SENTRY_SCHEMA_VERSION_NAME) + results.discard(POLICY_SENTRY_SCHEMA_VERSION_NAME) return results diff --git a/policy_sentry/querying/all_v1.py b/policy_sentry/querying/all_v1.py index 983ec2e42..3b0640338 100644 --- a/policy_sentry/querying/all_v1.py +++ b/policy_sentry/querying/all_v1.py @@ -17,7 +17,9 @@ def get_all_actions_v1( :param lowercase: Set to true to have the list of actions be in all lowercase strings. :return: A list of all actions present in the database. """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) all_actions = set() diff --git a/policy_sentry/querying/arns.py b/policy_sentry/querying/arns.py index 349f9137f..b47d09e5b 100644 --- a/policy_sentry/querying/arns.py +++ b/policy_sentry/querying/arns.py @@ -5,16 +5,16 @@ from __future__ import annotations -import logging import functools +import logging import warnings from typing import Any, cast from policy_sentry.querying.arns_v1 import get_arn_type_details_v1 from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2 from policy_sentry.shared.iam_data import ( - get_service_prefix_data, get_iam_definition_schema_version, + get_service_prefix_data, ) from policy_sentry.util.arns import does_arn_match, get_service_from_arn @@ -33,11 +33,13 @@ def get_arn_data(service_prefix: str, resource_type_name: str) -> list[dict[str, Returns: Dictionary: Metadata about an ARN type """ - warnings.warn("Please use get_arn_type_details() instead", DeprecationWarning) + warnings.warn( + "Please use get_arn_type_details() instead", DeprecationWarning, stacklevel=2 + ) results = [] service_prefix_data = get_service_prefix_data(service_prefix) - for resource_name, resource_data in service_prefix_data["resources"].items(): + for resource_data in service_prefix_data["resources"].values(): if resource_data["resource"].lower() == resource_type_name.lower(): output = { "resource_type_name": resource_data["resource"], diff --git a/policy_sentry/querying/arns_v1.py b/policy_sentry/querying/arns_v1.py index e8ef2efae..40b1b9e30 100644 --- a/policy_sentry/querying/arns_v1.py +++ b/policy_sentry/querying/arns_v1.py @@ -20,11 +20,13 @@ def get_arn_type_details_v1( Returns: Dictionary: Metadata about an ARN type """ - warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning) + warnings.warn( + "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2 + ) service_prefix_data = get_service_prefix_data(service_prefix) output = {} - for resource_name, resource_data in service_prefix_data["resources"].items(): + for resource_data in service_prefix_data["resources"].values(): if resource_data["resource"].lower() == resource_type_name.lower(): output = { "resource_type_name": resource_data["resource"], diff --git a/policy_sentry/querying/conditions.py b/policy_sentry/querying/conditions.py index 9e43e4e9e..64e61a3f6 100644 --- a/policy_sentry/querying/conditions.py +++ b/policy_sentry/querying/conditions.py @@ -5,13 +5,13 @@ from __future__ import annotations -import logging import functools +import logging from typing import cast +from policy_sentry.querying.actions import get_action_data from policy_sentry.shared.iam_data import get_service_prefix_data from policy_sentry.util.conditions import is_condition_key_match -from policy_sentry.querying.actions import get_action_data logger = logging.getLogger(__name__) @@ -107,7 +107,7 @@ def get_condition_value_type(condition_key: str) -> str | None: Returns: String: type of the condition key, like Bool, Date, String, etc. """ - service_prefix, condition_name = condition_key.split(":") + service_prefix, _ = condition_key.split(":") service_prefix_data = get_service_prefix_data(service_prefix) for condition_key_entry, condition_key_data in service_prefix_data[ diff --git a/policy_sentry/shared/awsdocs.py b/policy_sentry/shared/awsdocs.py index 9a39739dd..8b10b44b0 100644 --- a/policy_sentry/shared/awsdocs.py +++ b/policy_sentry/shared/awsdocs.py @@ -11,22 +11,22 @@ from __future__ import annotations -import os +import json import logging +import os import re -import json from pathlib import Path from typing import Any import requests -from bs4 import BeautifulSoup, Tag, PageElement +from bs4 import BeautifulSoup, PageElement, Tag from policy_sentry.shared.constants import ( BASE_DOCUMENTATION_URL, BUNDLED_ACCESS_OVERRIDES_FILE, LOCAL_HTML_DIRECTORY_PATH, - POLICY_SENTRY_SCHEMA_VERSION_NAME, POLICY_SENTRY_SCHEMA_VERSION_LATEST, + POLICY_SENTRY_SCHEMA_VERSION_NAME, ) from policy_sentry.util.access_levels import determine_access_level_override from policy_sentry.util.file import read_yaml_file @@ -42,9 +42,7 @@ def header_matches(string: str, table: Tag) -> bool: if string in header: match_found = True break - if not match_found: - return False - return True + return match_found def get_links_from_base_actions_resources_conditions_page() -> list[str]: @@ -121,13 +119,12 @@ def update_html_docs_directory(html_docs_destination: str) -> None: for script in soup.find_all("script"): try: - if "src" in script.attrs: - if script.get("src").startswith("/"): - temp = script.attrs["src"] - script.attrs["src"] = script.attrs["src"].replace( - temp, f"https://docs.aws.amazon.com{temp}" - ) - except TypeError as t_e: + if "src" in script.attrs and script.get("src").startswith("/"): + temp = script.attrs["src"] + script.attrs["src"] = script.attrs["src"].replace( + temp, f"https://docs.aws.amazon.com{temp}" + ) + except TypeError as t_e: # noqa: PERF203 logger.warning(t_e) logger.warning(script) except AttributeError as a_e: @@ -180,277 +177,276 @@ def create_database( # for filename in ['list_amazonathena.partial.html']: file_list = [] for filename in os.listdir(LOCAL_HTML_DIRECTORY_PATH): - if os.path.isfile(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename)): - if filename not in file_list: - file_list.append(filename) + if ( + os.path.isfile(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename)) + and filename not in file_list + ): + file_list.append(filename) file_list.sort() for filename in file_list: if not filename.startswith("list_"): continue - with open(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename)) as f: - soup = BeautifulSoup(f.read(), "html.parser") - main_content = soup.find(id="main-content") - if not isinstance(main_content, Tag): - continue + content = (Path(LOCAL_HTML_DIRECTORY_PATH) / filename).read_text() + soup = BeautifulSoup(content, "html.parser") + main_content = soup.find(id="main-content") + if not isinstance(main_content, Tag): + continue - # Get service name - topic_title = main_content.find("h1", class_="topictitle") - if not isinstance(topic_title, PageElement): - continue + # Get service name + topic_title = main_content.find("h1", class_="topictitle") + if not isinstance(topic_title, PageElement): + continue - title = re.sub( - ".*Actions, resources, and condition Keys for *", - "", - topic_title.text, - flags=re.IGNORECASE, - ) - title = title.replace("", "") - service_name = chomp(title) + title = re.sub( + ".*Actions, resources, and condition Keys for *", + "", + topic_title.text, + flags=re.IGNORECASE, + ) + title = title.replace("", "") + service_name = chomp(title) - service_prefix = "" - title_parent = topic_title.parent - if title_parent is None: - continue + service_prefix = "" + title_parent = topic_title.parent + if title_parent is None: + continue - for c in title_parent.children: - if "prefix" in str(c): - service_prefix = str(c) - service_prefix = service_prefix.split('')[1] - service_prefix = chomp(service_prefix.split("")[0]) - break + for c in title_parent.children: + if "prefix" in str(c): + service_prefix = str(c) + service_prefix = service_prefix.split('')[1] + service_prefix = chomp(service_prefix.split("")[0]) + break + + if service_prefix not in schema: + # The URL to that service's Actions, Resources, and Condition Keys page + service_authorization_url_prefix = ( + "https://docs.aws.amazon.com/service-authorization/latest/reference" + ) + service_authorization_url = f"{service_authorization_url_prefix}/{filename}" + schema[service_prefix] = { + "service_name": service_name, + "prefix": service_prefix, + "service_authorization_url": service_authorization_url, + "privileges": {}, + "privileges_lower_name": {}, # used for faster lookups + "resources": {}, + "resources_lower_name": {}, # used for faster lookups + "conditions": {}, + } + + access_level_overrides_cfg = get_action_access_level_overrides_from_yml( + service_prefix, access_level_overrides_file + ) - if service_prefix not in schema: - # The URL to that service's Actions, Resources, and Condition Keys page - service_authorization_url_prefix = ( - "https://docs.aws.amazon.com/service-authorization/latest/reference" - ) - service_authorization_url = ( - f"{service_authorization_url_prefix}/{filename}" - ) - schema[service_prefix] = { - "service_name": service_name, - "prefix": service_prefix, - "service_authorization_url": service_authorization_url, - "privileges": {}, - "privileges_lower_name": {}, # used for faster lookups - "resources": {}, - "resources_lower_name": {}, # used for faster lookups - "conditions": {}, - } + tables = main_content.find_all("div", class_="table-contents") - access_level_overrides_cfg = get_action_access_level_overrides_from_yml( - service_prefix, access_level_overrides_file - ) + for table in tables: + # There can be 3 tables, the actions table, an ARN table, and a condition key table + # Example: https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awssecuritytokenservice.html + if not header_matches("actions", table) or not header_matches( + "description", table + ): + continue - tables = main_content.find_all("div", class_="table-contents") + rows = table.find_all("tr") + row_number = 0 + while row_number < len(rows): + row = rows[row_number] - for table in tables: - # There can be 3 tables, the actions table, an ARN table, and a condition key table - # Example: https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awssecuritytokenservice.html - if not header_matches("actions", table) or not header_matches( - "description", table - ): + cells = row.find_all("td") + if len(cells) == 0: + # Skip the header row, which has th, not td cells + row_number += 1 continue - rows = table.find_all("tr") - row_number = 0 - while row_number < len(rows): - row = rows[row_number] + if len(cells) != 6: + # Sometimes the privilege contains Scenarios, and I don't know how to handle this + # raise Exception("Unexpected format in {}: {}".format(prefix, row)) + break - cells = row.find_all("td") - if len(cells) == 0: - # Skip the header row, which has th, not td cells - row_number += 1 + # See if this cell spans multiple rows + rowspan = 1 + if "rowspan" in cells[0].attrs: + rowspan = int(cells[0].attrs["rowspan"]) + + priv = "" + # Get the privilege + for link in cells[0].find_all("a"): + if "href" not in link.attrs: # pylint: disable=no-else-continue + # Skip the tags + api_documentation_link = None continue - - if len(cells) != 6: - # Sometimes the privilege contains Scenarios, and I don't know how to handle this - # raise Exception("Unexpected format in {}: {}".format(prefix, row)) - break - - # See if this cell spans multiple rows - rowspan = 1 - if "rowspan" in cells[0].attrs: - rowspan = int(cells[0].attrs["rowspan"]) - - priv = "" - # Get the privilege - for link in cells[0].find_all("a"): - if "href" not in link.attrs: # pylint: disable=no-else-continue - # Skip the tags - api_documentation_link = None - continue - else: - api_documentation_link = link.attrs.get("href") - logger.debug(api_documentation_link) - priv = chomp(link.text) - if priv == "": - priv = chomp(cells[0].text) - action_name = priv - description = chomp(cells[1].text) - access_level = chomp(cells[2].text) - # Access Level ##### - # access_level_overrides_cfg will only be true if the service in question is present - # in the overrides YML file - if access_level_overrides_cfg: - override_result = determine_access_level_override( - service=service_prefix, - action_name=action_name, - provided_access_level=access_level, - service_override_config=access_level_overrides_cfg, + else: + api_documentation_link = link.attrs.get("href") + logger.debug(api_documentation_link) + priv = chomp(link.text) + if priv == "": + priv = chomp(cells[0].text) + action_name = priv + description = chomp(cells[1].text) + access_level = chomp(cells[2].text) + # Access Level ##### + # access_level_overrides_cfg will only be true if the service in question is present + # in the overrides YML file + if access_level_overrides_cfg: + override_result = determine_access_level_override( + service=service_prefix, + action_name=action_name, + provided_access_level=access_level, + service_override_config=access_level_overrides_cfg, + ) + if override_result: + access_level = override_result + logger.debug( + "Override: Setting access level for %s:%s to %s", + service_prefix, + action_name, + access_level, ) - if override_result: - access_level = override_result - logger.debug( - "Override: Setting access level for %s:%s to %s", - service_prefix, - action_name, - access_level, - ) - # else: - # access_level = access_level - # else: - # access_level = access_level - resource_types = {} - resource_types_lower_name = {} - resource_cell = 3 - - while rowspan > 0: - if len(cells) == 3 or len(cells) == 6: - # ec2:RunInstances contains a few "scenarios" which start in the - # description field, len(cells) is 5. - # I'm ignoring these as I don't know how to handle them. - # These include things like "EC2-Classic-InstanceStore" and - # "EC2-VPC-InstanceStore-Subnet" - - resource_type = chomp(cells[resource_cell].text) - - condition_keys_element = cells[resource_cell + 1] - condition_keys = [] - if condition_keys_element.text != "": - for key_element in condition_keys_element.find_all("p"): - condition_keys.append(chomp(key_element.text)) - - dependent_actions_element = cells[resource_cell + 2] - dependent_actions = [] - if dependent_actions_element.text != "": - for ( - action_element - ) in dependent_actions_element.find_all("p"): - chomped_action = chomp(action_element.text) - dependent_actions.append( - sanitize_service_name(chomped_action) - ) - if "*" in resource_type: - required = True - resource_type = resource_type.strip("*") - else: - required = False - - resource_types[resource_type] = { - "resource_type": resource_type, - "required": required, - "condition_keys": condition_keys, - "dependent_actions": dependent_actions, - } - resource_types_lower_name[resource_type.lower()] = ( - resource_type - ) - rowspan -= 1 - if rowspan > 0: - row_number += 1 - resource_cell = 0 - row = rows[row_number] - cells = row.find_all("td") - - if "[permission only]" in priv: - priv = priv.split(" ", maxsplit=1)[0] - - privilege_schema = { - "privilege": priv, - "description": description, - "access_level": access_level, - "resource_types": resource_types, - "resource_types_lower_name": resource_types_lower_name, - "api_documentation_link": api_documentation_link, - } - - schema[service_prefix]["privileges"][priv] = privilege_schema - schema[service_prefix]["privileges_lower_name"][priv.lower()] = priv - row_number += 1 + # else: + # access_level = access_level + # else: + # access_level = access_level + resource_types = {} + resource_types_lower_name = {} + resource_cell = 3 + + while rowspan > 0: + if len(cells) == 3 or len(cells) == 6: + # ec2:RunInstances contains a few "scenarios" which start in the + # description field, len(cells) is 5. + # I'm ignoring these as I don't know how to handle them. + # These include things like "EC2-Classic-InstanceStore" and + # "EC2-VPC-InstanceStore-Subnet" + + resource_type = chomp(cells[resource_cell].text) + + condition_keys_element = cells[resource_cell + 1] + condition_keys = [ + chomp(key_element.text) + for key_element in condition_keys_element.find_all("p") + if condition_keys_element.text != "" + ] + + dependent_actions_element = cells[resource_cell + 2] + dependent_actions = [] + if dependent_actions_element.text != "": + for action_element in dependent_actions_element.find_all( + "p" + ): + chomped_action = chomp(action_element.text) + dependent_actions.append( + sanitize_service_name(chomped_action) + ) + if "*" in resource_type: + required = True + resource_type = resource_type.strip("*") + else: + required = False + + resource_types[resource_type] = { + "resource_type": resource_type, + "required": required, + "condition_keys": condition_keys, + "dependent_actions": dependent_actions, + } + resource_types_lower_name[resource_type.lower()] = resource_type + rowspan -= 1 + if rowspan > 0: + row_number += 1 + resource_cell = 0 + row = rows[row_number] + cells = row.find_all("td") + + if "[permission only]" in priv: + priv = priv.split(" ", maxsplit=1)[0] + + privilege_schema = { + "privilege": priv, + "description": description, + "access_level": access_level, + "resource_types": resource_types, + "resource_types_lower_name": resource_types_lower_name, + "api_documentation_link": api_documentation_link, + } + + schema[service_prefix]["privileges"][priv] = privilege_schema + schema[service_prefix]["privileges_lower_name"][priv.lower()] = priv + row_number += 1 + + # Get resource table + for table in tables: + if not header_matches("resource types", table) or not header_matches( + "arn", table + ): + continue + + rows = table.find_all("tr") + for row in rows: + cells = row.find_all("td") - # Get resource table - for table in tables: - if not header_matches("resource types", table) or not header_matches( - "arn", table - ): + if len(cells) == 0: + # Skip the header row, which has th, not td cells continue - rows = table.find_all("tr") - for row in rows: - cells = row.find_all("td") + if len(cells) != 3: + raise Exception( + f"Unexpected number of resource cells {len(cells)} in {filename}" + ) - if len(cells) == 0: - # Skip the header row, which has th, not td cells - continue + resource = chomp(cells[0].text) - if len(cells) != 3: - raise Exception( - f"Unexpected number of resource cells {len(cells)} in {filename}" - ) + arn = no_white_space(cells[1].text) + conditions = [ + chomp(condition.text) for condition in cells[2].find_all("p") + ] - resource = chomp(cells[0].text) + schema[service_prefix]["resources"][resource] = { + "resource": resource, + "arn": arn, + "condition_keys": conditions, + } + schema[service_prefix]["resources_lower_name"][resource.lower()] = ( + resource + ) - arn = no_white_space(cells[1].text) - conditions = [] - for condition in cells[2].find_all("p"): - conditions.append(chomp(condition.text)) + # Get condition keys table + for table in tables: + if not ( + header_matches(" condition keys ", table) + and header_matches(" type ", table) + ): + continue - schema[service_prefix]["resources"][resource] = { - "resource": resource, - "arn": arn, - "condition_keys": conditions, - } - schema[service_prefix]["resources_lower_name"][resource.lower()] = ( - resource - ) + rows = table.find_all("tr") + for row in rows: + cells = row.find_all("td") - # Get condition keys table - for table in tables: - if not ( - header_matches(" condition keys ", table) - and header_matches(" type ", table) - ): + if len(cells) == 0: + # Skip the header row, which has th, not td cells continue - rows = table.find_all("tr") - for row in rows: - cells = row.find_all("td") - - if len(cells) == 0: - # Skip the header row, which has th, not td cells - continue + if len(cells) != 3: + raise Exception( + f"Unexpected number of condition cells {len(cells)} in {filename}" + ) - if len(cells) != 3: - raise Exception( - f"Unexpected number of condition cells {len(cells)} in {filename}" - ) + condition = no_white_space(cells[0].text) + description = chomp(cells[1].text) + value_type = chomp(cells[2].text) - condition = no_white_space(cells[0].text) - description = chomp(cells[1].text) - value_type = chomp(cells[2].text) - - schema[service_prefix]["conditions"][condition] = { - "condition": condition, - "description": description, - "type": value_type, - } - # this_service_schema = { - # service_prefix: service_schema - # } - # schema.update(this_service_schema) + schema[service_prefix]["conditions"][condition] = { + "condition": condition, + "description": description, + "type": value_type, + } + # this_service_schema = { + # service_prefix: service_schema + # } + # schema.update(this_service_schema) iam_definition_file = os.path.join(destination_directory, "iam-definition.json") with open(iam_definition_file, "w") as file: diff --git a/policy_sentry/shared/constants.py b/policy_sentry/shared/constants.py index 12d07c7ab..6330b9766 100644 --- a/policy_sentry/shared/constants.py +++ b/policy_sentry/shared/constants.py @@ -2,9 +2,9 @@ Just a common storage space for storing some constants. """ -from pathlib import Path -import os import logging +import os +from pathlib import Path logger = logging.getLogger() diff --git a/policy_sentry/shared/iam_data.py b/policy_sentry/shared/iam_data.py index 6011bba8e..fb3dcc230 100644 --- a/policy_sentry/shared/iam_data.py +++ b/policy_sentry/shared/iam_data.py @@ -2,9 +2,9 @@ from __future__ import annotations +import functools import json import logging -import functools from pathlib import Path from typing import Any, cast @@ -47,8 +47,7 @@ def get_service_prefix_data(service_prefix: str) -> dict[str, Any]: """ try: return cast("dict[str, Any]", iam_definition[service_prefix]) - # pylint: disable=bare-except, inconsistent-return-statements - except: + except: # noqa: E722 if service_prefix == "catalog": # the resource types "Portfolio" and "Product" have the service name "catalog" in their ARN # https://docs.aws.amazon.com/service-authorization/latest/reference/list_awsservicecatalog.html#awsservicecatalog-resources-for-iam-policies diff --git a/policy_sentry/util/access_levels.py b/policy_sentry/util/access_levels.py index 4fb220e29..f80294f14 100644 --- a/policy_sentry/util/access_levels.py +++ b/policy_sentry/util/access_levels.py @@ -4,8 +4,8 @@ from __future__ import annotations -import sys import logging +import sys logger = logging.getLogger(__name__) @@ -38,11 +38,10 @@ def override_access_level( ) # first index will contain the access level given in the override config for that action. # since we break the loop, we know it only contains one value. - if real_access_level: + if real_access_level and real_access_level != provided_access_level: # If AWS hasn't fixed their documentation yet, then our YAML override cfg will not match their documentation. # Therefore, accept our override instead. - if real_access_level != provided_access_level: - return real_access_level + return real_access_level # Otherwise, they have fixed their documentation because our override file matches their documentation. # Therefore, return false because we don't need to override diff --git a/policy_sentry/util/arns.py b/policy_sentry/util/arns.py index 7083e81c2..996191fff 100644 --- a/policy_sentry/util/arns.py +++ b/policy_sentry/util/arns.py @@ -136,9 +136,10 @@ def same_resource_type(self, arn_in_database: str) -> bool: # and length of the arn_format_list should be the same as split_resource_string_to_test # If all conditions match, then the ARN format is the same. if elem: - if elem == split_resource_string_to_test[idx]: - pass - elif split_resource_string_to_test[idx] == "*": + if ( + elem == split_resource_string_to_test[idx] + or split_resource_string_to_test[idx] == "*" + ): pass else: return False @@ -158,10 +159,10 @@ def same_resource_type(self, arn_in_database: str) -> bool: # If we've made it this far, then it is a special type # return True # Presence of / would mean it's an object in both so it matches - elif "/" in self.resource_string and "/" in elements[5]: + elif "/" in self.resource_string and "/" in elements[5]: # noqa: SIM114 return True # / not being present in either means it's a bucket in both so it matches - elif "/" not in self.resource_string and "/" not in elements[5]: + elif "/" not in self.resource_string and "/" not in elements[5]: # noqa: SIM103 return True # If there is a / in one but not in the other, it does not match else: diff --git a/policy_sentry/util/conditions.py b/policy_sentry/util/conditions.py index 8852b802f..397c4ad53 100644 --- a/policy_sentry/util/conditions.py +++ b/policy_sentry/util/conditions.py @@ -16,13 +16,13 @@ def translate_condition_key_data_types(condition_str: str) -> str: return "Arn" elif condition_lowercase in ("bool", "boolean"): return "Bool" - elif condition_lowercase in ("date",): + elif condition_lowercase == "date": return "Date" elif condition_lowercase in ("long", "numeric"): return "Number" elif condition_lowercase in ("string", "string", "arrayofstring"): return "String" - elif condition_lowercase in ("ip",): + elif condition_lowercase == "ip": return "Ip" else: raise Exception(f"Unknown data format: {condition_lowercase}") @@ -63,7 +63,7 @@ def is_condition_key_match(document_key: str, some_str: str) -> bool: # Some services use a format like s3:ExistingObjectTag/ if some_str.startswith(document_key.split("<")[0]): return True - elif "tag-key" in document_key: + elif "tag-key" in document_key: # noqa: SIM102 # Some services use a format like secretsmanager:ResourceTag/tag-key if some_str.startswith(document_key.split("tag-key")[0]): return True diff --git a/policy_sentry/util/policy_files.py b/policy_sentry/util/policy_files.py index 064c2824b..91e72d1f9 100644 --- a/policy_sentry/util/policy_files.py +++ b/policy_sentry/util/policy_files.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Any - from policy_sentry.querying.actions import get_action_data logger = logging.getLogger(__name__) @@ -54,8 +53,9 @@ def get_actions_from_policy(data: dict[str, Any]) -> list[str]: for action in actions_list: service, action_name = action.split(":") action_data = get_action_data(service, action_name) - if service in action_data and action_data[service]: - new_actions_list.append(action_data[service][0]["action"]) + service_data = action_data.get(service) + if service_data: + new_actions_list.append(service_data[0]["action"]) new_actions_list.sort() return new_actions_list @@ -74,8 +74,7 @@ def get_actions_from_json_policy_file(file: str | Path) -> list[str]: # in this tool. [MJ] data = json.load(json_file) actions_list = get_actions_from_policy(data) - - except: # pylint: disable=bare-except + except: # noqa: E722 logger.debug("General Error at get_actions_from_json_policy_file.") actions_list = [] return actions_list diff --git a/policy_sentry/writing/minimize.py b/policy_sentry/writing/minimize.py index 1a5830843..6c8a6becd 100644 --- a/policy_sentry/writing/minimize.py +++ b/policy_sentry/writing/minimize.py @@ -24,8 +24,8 @@ from __future__ import annotations -import logging import functools +import logging logger = logging.getLogger(__name__) @@ -69,15 +69,7 @@ def check_min_permission_length(permission: str, minchars: int | None = None) -> Adapted version of policyuniverse's _check_permission_length. We are commenting out the skipping prefix message https://github.com/Netflix-Skunkworks/policyuniverse/blob/master/policyuniverse/expander_minimizer.py#L111 """ - if minchars and permission and len(permission) < int(minchars): - # print( - # "Skipping prefix {} because length of {}".format( - # permission, len(permission) - # ), - # file=sys.stderr, - # ) - return True - return False + return bool(minchars and permission and len(permission) < int(minchars)) # This is a condensed version of policyuniverse's minimize_statement_actions, changed for our purposes. @@ -115,13 +107,12 @@ def minimize_statement_actions( if check_min_permission_length(permission, minchars=minchars): continue # If the action name is not empty - if prefix not in denied_prefixes: - if permission: - if prefix not in desired_actions: - prefix = f"{prefix}*" - minimized_actions.add(prefix) - found_prefix = True - break + if prefix not in denied_prefixes and permission: + if prefix not in desired_actions: + prefix = f"{prefix}*" + minimized_actions.add(prefix) + found_prefix = True + break if not found_prefix: logger.debug( diff --git a/policy_sentry/writing/sid_group.py b/policy_sentry/writing/sid_group.py index 893cf0b25..f07aa8bbf 100644 --- a/policy_sentry/writing/sid_group.py +++ b/policy_sentry/writing/sid_group.py @@ -9,21 +9,21 @@ from typing import Any from policy_sentry.analysis.expand import determine_actions_to_expand -from policy_sentry.querying.all import get_all_actions from policy_sentry.querying.actions import ( get_action_data, + get_actions_at_access_level_that_support_wildcard_arns_only, + get_actions_that_support_wildcard_arns_only, get_actions_with_arn_type_and_access_level, get_dependent_actions, - get_actions_that_support_wildcard_arns_only, - get_actions_at_access_level_that_support_wildcard_arns_only, ) +from policy_sentry.querying.all import get_all_actions from policy_sentry.querying.arns import get_resource_type_name_with_raw_arn +from policy_sentry.shared.constants import POLICY_LANGUAGE_VERSION +from policy_sentry.util.actions import get_lowercase_action_list from policy_sentry.util.arns import does_arn_match, get_service_from_arn, parse_arn from policy_sentry.util.text import capitalize_first_character, strip_special_characters from policy_sentry.writing.minimize import minimize_statement_actions from policy_sentry.writing.validate import check_actions_schema, check_crud_schema -from policy_sentry.shared.constants import POLICY_LANGUAGE_VERSION -from policy_sentry.util.actions import get_lowercase_action_list SANITIZE_NAME_PATTERN = re.compile(r"[^A-Za-z0-9]+") @@ -308,7 +308,7 @@ def add_by_arn_and_access_level( self, arn_list: list[str], access_level: str, - conditions_block: dict[str, Any] | None = None, + _conditions_block: dict[str, Any] | None = None, ) -> None: """ This adds the user-supplied ARN(s), service prefixes, access levels, and condition keys (if applicable) given @@ -426,8 +426,8 @@ def add_by_list_of_actions(self, supplied_actions: list[str]) -> dict[str, Any]: if action not in supplied_actions ] logger.debug("Adding by list of actions") - logger.debug(f"Supplied actions: {str(supplied_actions)}") - logger.debug(f"Dependent actions: {str(dependent_actions)}") + logger.debug(f"Supplied actions: {supplied_actions}") + logger.debug(f"Dependent actions: {dependent_actions}") arns_matching_supplied_actions = [] # arns_matching_supplied_actions is a list of dicts. @@ -471,8 +471,8 @@ def add_by_list_of_actions(self, supplied_actions: list[str]) -> dict[str, Any]: logger.debug( "Purging actions that do not match the requested actions and dependent actions" ) - logger.debug(f"Supplied actions: {str(supplied_actions)}") - logger.debug(f"Dependent actions: {str(dependent_actions)}") + logger.debug(f"Supplied actions: {supplied_actions}") + logger.debug(f"Dependent actions: {dependent_actions}") self.remove_actions_not_matching_these(supplied_actions + dependent_actions) for action in actions_without_resource_constraints: logger.debug( @@ -515,36 +515,28 @@ def process_template( provided_wildcard_actions = cfg_wildcard.get("single-actions") if provided_wildcard_actions and provided_wildcard_actions[0] != "": logger.debug( - f"Requested wildcard-only actions: {str(provided_wildcard_actions)}" + f"Requested wildcard-only actions: {provided_wildcard_actions}" ) self.wildcard_only_single_actions = provided_wildcard_actions service_read = cfg_wildcard.get("service-read") if service_read and service_read[0]: - logger.debug( - f"Requested wildcard-only actions: {str(service_read)}" - ) + logger.debug(f"Requested wildcard-only actions: {service_read}") self.wildcard_only_service_read = service_read service_write = cfg_wildcard.get("service-write") if service_write and service_write[0]: - logger.debug( - f"Requested wildcard-only actions: {str(service_write)}" - ) + logger.debug(f"Requested wildcard-only actions: {service_write}") self.wildcard_only_service_write = service_write service_list = cfg_wildcard.get("service-list") if service_list and service_list[0]: - logger.debug( - f"Requested wildcard-only actions: {str(service_list)}" - ) + logger.debug(f"Requested wildcard-only actions: {service_list}") self.wildcard_only_service_list = service_list service_tagging = cfg_wildcard.get("service-tagging") if service_tagging and service_tagging[0]: - logger.debug( - f"Requested wildcard-only actions: {str(service_tagging)}" - ) + logger.debug(f"Requested wildcard-only actions: {service_tagging}") self.wildcard_only_service_tagging = service_tagging service_permissions_management = cfg_wildcard.get( @@ -552,7 +544,7 @@ def process_template( ) if service_permissions_management and service_permissions_management[0]: logger.debug( - f"Requested wildcard-only actions: {str(service_permissions_management)}" + f"Requested wildcard-only actions: {service_permissions_management}" ) self.wildcard_only_service_permissions_management = ( service_permissions_management @@ -564,30 +556,30 @@ def process_template( # Standard access levels cfg_read = cfg.get("read") if cfg_read and cfg_read[0]: - logger.debug(f"Requested access to arns: {str(cfg_read)}") + logger.debug(f"Requested access to arns: {cfg_read}") self.add_by_arn_and_access_level(cfg_read, "Read") cfg_write = cfg.get("write") if cfg_write and cfg_write[0]: - logger.debug(f"Requested access to arns: {str(cfg_write)}") + logger.debug(f"Requested access to arns: {cfg_write}") self.add_by_arn_and_access_level(cfg_write, "Write") cfg_list = cfg.get("list") if cfg_list and cfg_list[0]: - logger.debug(f"Requested access to arns: {str(cfg_list)}") + logger.debug(f"Requested access to arns: {cfg_list}") self.add_by_arn_and_access_level(cfg_list, "List") tagging = cfg.get("tagging") if tagging and tagging[0]: - logger.debug(f"Requested access to arns: {str(tagging)}") + logger.debug(f"Requested access to arns: {tagging}") self.add_by_arn_and_access_level(tagging, "Tagging") cfg_mgmt = cfg.get("permissions-management") if cfg_mgmt and cfg_mgmt[0]: - logger.debug(f"Requested access to arns: {str(cfg_mgmt)}") + logger.debug(f"Requested access to arns: {cfg_mgmt}") self.add_by_arn_and_access_level(cfg_mgmt, "Permissions management") # SKIP RESOURCE CONSTRAINTS cfg_skip = cfg.get("skip-resource-constraints") if cfg_skip and cfg_skip[0]: logger.debug( - f"Requested override: the actions {str(cfg_skip)} will " + f"Requested override: the actions {cfg_skip} will " f"skip resource constraints." ) self.add_skip_resource_constraints(cfg_skip) @@ -600,13 +592,13 @@ def process_template( cfg_sts = cfg.get("sts") if cfg_sts: logger.debug( - f"STS section detected. Building assume role policy statement" + "STS section detected. Building assume role policy statement" ) self.add_sts_actions(cfg_sts) elif cfg_mode == "actions": check_actions_schema(cfg) - if "actions" in cfg.keys(): + if "actions" in cfg: cfg_actions = cfg["actions"] if cfg_actions is not None and cfg_actions[0] != "": self.add_by_list_of_actions(cfg_actions) @@ -657,7 +649,7 @@ def remove_actions_not_matching_these(self, actions_to_keep: list[str]) -> None: """ actions_to_keep = get_lowercase_action_list(actions_to_keep) actions_deleted = [] - for sid, group in self.sids.items(): + for group in self.sids.values(): placeholder_actions_list = [] for action in group["actions"]: # if the action is not in the list of selected actions, don't copy it to the placeholder list @@ -712,10 +704,12 @@ def remove_actions_duplicated_in_wildcard_arn(self) -> None: for group in self.sids.values(): if "*" not in group["arn_format"]: for action in actions_under_wildcard_resources: - if action in group["actions"]: - if action not in self.skip_resource_constraints: - # add it to a list of actions to nuke when they are under other SIDs - actions_under_wildcard_resources_to_nuke.append(action) + if ( + action in group["actions"] + and action not in self.skip_resource_constraints + ): + # add it to a list of actions to nuke when they are under other SIDs + actions_under_wildcard_resources_to_nuke.append(action) # noqa: PERF401 # If there are actions that we need to remove from SIDs outside of MultMultNone SID if actions_under_wildcard_resources_to_nuke: @@ -741,7 +735,7 @@ def remove_actions_that_are_not_wildcard_arn_only(actions_list: list[str]) -> li for action in actions_set: try: - service_name, action_name = action.split(":") + service_name, _ = action.split(":") except ValueError as v_e: # We will skip the action because this likely means that the wildcard action provided is not valid. logger.debug(v_e) @@ -754,7 +748,7 @@ def remove_actions_that_are_not_wildcard_arn_only(actions_list: list[str]) -> li rows = get_actions_that_support_wildcard_arns_only(service_name) for row in rows: if row.lower() == action_lower: - actions_list_placeholder.append(action) + actions_list_placeholder.append(action) # noqa: PERF401 return actions_list_placeholder diff --git a/policy_sentry/writing/validate.py b/policy_sentry/writing/validate.py index 0ffd25512..cadbf0b35 100644 --- a/policy_sentry/writing/validate.py +++ b/policy_sentry/writing/validate.py @@ -7,7 +7,7 @@ import logging from typing import Any -from schema import Optional, Schema, And, Use, Regex, SchemaError +from schema import And, Optional, Regex, Schema, SchemaError, Use logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ def check(conf_schema: Schema, conf: dict[str, Any]) -> bool: detailed_error_message = schema_error.autos[2] print(detailed_error_message.split(" in {'")[0]) # for error in schema_error.autos: - except: # pylint: disable=bare-except + except: # noqa: E722 logger.critical(schema_error) return False @@ -80,9 +80,9 @@ def check_actions_schema(cfg: dict[str, Any]) -> bool: return result else: raise Exception( - f"The provided template does not match the required schema for ACTIONS mode. " - f"Please use the create-template command to generate a valid YML template that " - f"Policy Sentry will accept." + "The provided template does not match the required schema for ACTIONS mode. " + "Please use the create-template command to generate a valid YML template that " + "Policy Sentry will accept." ) @@ -95,9 +95,9 @@ def check_crud_schema(cfg: dict[str, Any]) -> bool: return result else: raise Exception( - f"The provided template does not match the required schema for CRUD mode. " - f"Please use the create-template command to generate a valid YML template that " - f"Policy Sentry will accept." + "The provided template does not match the required schema for CRUD mode. " + "Please use the create-template command to generate a valid YML template that " + "Policy Sentry will accept." ) @@ -112,7 +112,7 @@ def validate_condition_block(condition_block: dict[str, Any]) -> bool: """ # TODO: Validate that the values are legit somehow - CONDITION_BLOCK_SCHEMA = Schema( + condition_block_schema = Schema( { "condition_key_string": And(Use(str)), "condition_type_string": And(Use(str)), @@ -120,7 +120,7 @@ def validate_condition_block(condition_block: dict[str, Any]) -> bool: } ) try: - CONDITION_BLOCK_SCHEMA.validate(condition_block) + condition_block_schema.validate(condition_block) # TODO: Try to validate whether or not the condition keys are legit return True except SchemaError as s_e: diff --git a/pyproject.toml b/pyproject.toml index 995989d25..471378309 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,34 @@ module = [ ] ignore_missing_imports = true +[tool.ruff] +target-version = "py37" + +[tool.ruff.lint] +preview = true +select = [ + "A", + "ANN", + "ARG", + "B", + "E", + "F", + "FURB", + "I", + "ISC", + "N", + "PERF", + "PIE", + "RUF", + "S", + "SIM", + "T10", + "UP", + "W", + "YTT", +] +ignore = ["E501"] # ruff fromat takes care of it + [tool.pytest.ini_options] testpaths = [ "test", diff --git a/requirements-dev.txt b/requirements-dev.txt index c9de790c1..0cf938a9c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,13 +2,11 @@ pre-commit==2.21.0 # Unit Testing pytest==7.4.0 -pylint==2.17.5 coverage==7.2.7 # Integration tests and tasks invoke==2.2.0 # Security testing safety==2.3.5 -bandit>=1.7.5 # Type hints mypy==1.4.1 types-pyyaml==6.0.12.11 diff --git a/setup.py b/setup.py index f7297c8b9..0e65a39aa 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,29 @@ """Setup script for Policy Sentry""" + import setuptools import os import re HERE = os.path.abspath(os.path.dirname(__file__)) -VERSION_RE = re.compile(r'''__version__ = ['"]([0-9.]+)['"]''') -TESTS_REQUIRE = [ - 'coverage', - 'pytest' -] +VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") +TESTS_REQUIRE = ["coverage", "pytest"] REQUIRED_PACKAGES = [ - 'beautifulsoup4', - 'click', - 'requests', - 'schema', - 'PyYAML', + "beautifulsoup4", + "click", + "requests", + "schema", + "PyYAML", ] PROJECT_URLS = { "Documentation": "https://policy-sentry.readthedocs.io/", "Code": "https://github.com/salesforce/policy_sentry/", "Twitter": "https://twitter.com/kmcquade3", - "Red Team Report": "https://opensource.salesforce.com/policy_sentry" + "Red Team Report": "https://opensource.salesforce.com/policy_sentry", } def get_version(): - init = open( - os.path.join( - HERE, - "policy_sentry", - "bin", - "version.py" - ) - ).read() + init = open(os.path.join(HERE, "policy_sentry", "bin", "version.py")).read() return VERSION_RE.search(init).group(1) @@ -52,7 +43,7 @@ def get_description(): long_description=get_description(), long_description_content_type="text/markdown", url="https://github.com/salesforce/policy_sentry", - packages=setuptools.find_packages(exclude=['test*']), + packages=setuptools.find_packages(exclude=["test*"]), tests_require=TESTS_REQUIRE, install_requires=REQUIRED_PACKAGES, project_urls=PROJECT_URLS, @@ -70,6 +61,6 @@ def get_description(): ], entry_points={"console_scripts": "policy_sentry=policy_sentry.bin.cli:main"}, zip_safe=True, - keywords='aws iam roles policy policies privileges security', - python_requires='>=3.6', + keywords="aws iam roles policy policies privileges security", + python_requires=">=3.6", ) diff --git a/tasks.py b/tasks.py index 9e19024e5..ca81c88f8 100755 --- a/tasks.py +++ b/tasks.py @@ -3,13 +3,10 @@ import os import logging from invoke import task, Collection, UnexpectedExit, Failure + sys.path.append( os.path.abspath( - os.path.join( - os.path.dirname(__file__), - os.path.pardir, - 'policy_sentry' - ) + os.path.join(os.path.dirname(__file__), os.path.pardir, "policy_sentry") ) ) from policy_sentry.command import initialize @@ -18,52 +15,55 @@ # Create the necessary collections (namespaces) ns = Collection() -docs = Collection('docs') +docs = Collection("docs") ns.add_collection(docs) -test = Collection('test') +test = Collection("test") ns.add_collection(test) -integration = Collection('integration') +integration = Collection("integration") ns.add_collection(integration) -unit = Collection('unit') +unit = Collection("unit") ns.add_collection(unit) -build = Collection('build') +build = Collection("build") ns.add_collection(build) -docker = Collection('docker') +docker = Collection("docker") ns.add_collection(docker) @task def build_docs(c): """Create the documentation files and open them locally""" - c.run('mkdocs build') + c.run("mkdocs build") + @task def serve_docs(c): """Create the documentation files and open them locally""" c.run('mkdocs serve --dev-addr "127.0.0.1:8001"') + @task def download_latest_aws_docs(c): """Download the latest AWS docs, and update the bundled IAM database.""" - c.run('./utils/download_docs.py') + c.run("./utils/download_docs.py") + # BUILD @task def build_package(c): """Build the policy_sentry package from the current directory contents for use with PyPi""" - c.run('python -m pip install --upgrade setuptools wheel') - c.run('python setup.py -q sdist bdist_wheel') + c.run("python -m pip install --upgrade setuptools wheel") + c.run("python setup.py -q sdist bdist_wheel") @task(pre=[build_package]) def install_package(c): """Install the policy_sentry package built from the current directory contents (not PyPi)""" - c.run('pip3 install -q dist/policy_sentry-*.tar.gz') + c.run("pip3 install -q dist/policy_sentry-*.tar.gz") @task @@ -75,17 +75,21 @@ def uninstall_package(c): @task def upload_to_pypi_test_server(c): """Upload the package to the TestPyPi server (requires credentials)""" - c.run('python -m pip install --upgrade twine') - c.run('python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*') - c.run('python -m pip install --index-url https://test.pypi.org/simple/ --no-deps policy_sentry') + c.run("python -m pip install --upgrade twine") + c.run( + "python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*" + ) + c.run( + "python -m pip install --index-url https://test.pypi.org/simple/ --no-deps policy_sentry" + ) @task def upload_to_pypi_prod_server(c): """Upload the package to the PyPi production server (requires credentials)""" - c.run('python -m pip install --upgrade twine') - c.run('python -m twine upload dist/*') - c.run('python -m pip install policy_sentry') + c.run("python -m pip install --upgrade twine") + c.run("python -m twine upload dist/*") + c.run("python -m pip install policy_sentry") # INTEGRATION TESTS @@ -93,7 +97,7 @@ def upload_to_pypi_prod_server(c): def clean_config_directory(c): """Runs `rm -rf $HOME/.policy_sentry`""" try: - c.run('rm -rf $HOME/.policy_sentry/') + c.run("rm -rf $HOME/.policy_sentry/") except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -106,7 +110,7 @@ def clean_config_directory(c): def create_db(c): """Integration testing: Initialize the policy_sentry database""" try: - initialize.initialize('') + initialize.initialize("") except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -119,7 +123,7 @@ def create_db(c): def version_check(c): """Print the version""" try: - c.run('./policy_sentry/bin/cli.py --version', pty=True) + c.run("./policy_sentry/bin/cli.py --version", pty=True) except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -134,9 +138,18 @@ def write_policy(c): Integration testing: Tests the `write-policy` function. """ try: - c.run('./policy_sentry/bin/cli.py write-policy --input-file examples/yml/crud.yml', pty=True) - c.run('./policy_sentry/bin/cli.py write-policy --input-file examples/yml/crud.yml', pty=True) - c.run('./policy_sentry/bin/cli.py write-policy --input-file examples/yml/actions.yml', pty=True) + c.run( + "./policy_sentry/bin/cli.py write-policy --input-file examples/yml/crud.yml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py write-policy --input-file examples/yml/crud.yml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py write-policy --input-file examples/yml/actions.yml", + pty=True, + ) except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -150,22 +163,52 @@ def query(c): """Integration testing: Tests the `query` functionality (querying the IAM database)""" try: c.run('echo "Querying the action table"', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ram', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ram --name tagresource', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table ' - '--service ram --access-level permissions-management', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ssm --resource-type parameter', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ssm --access-level write ' - '--resource-type parameter', pty=True) - c.run('policy_sentry query action-table --service ssm --resource-type parameter', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ses --condition ses:FeedbackAddress', pty=True) + c.run("./policy_sentry/bin/cli.py query action-table --service ram", pty=True) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ram --name tagresource", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table " + "--service ram --access-level permissions-management", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ssm --resource-type parameter", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ssm --access-level write " + "--resource-type parameter", + pty=True, + ) + c.run( + "policy_sentry query action-table --service ssm --resource-type parameter", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ses --condition ses:FeedbackAddress", + pty=True, + ) c.run('echo "Querying the ARN table"', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service ssm', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service cloud9 --name environment', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service cloud9 --list-arn-types', pty=True) + c.run("./policy_sentry/bin/cli.py query arn-table --service ssm", pty=True) + c.run( + "./policy_sentry/bin/cli.py query arn-table --service cloud9 --name environment", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query arn-table --service cloud9 --list-arn-types", + pty=True, + ) c.run('echo "Querying the condition keys table"', pty=True) - c.run('./policy_sentry/bin/cli.py query condition-table --service cloud9', pty=True) - c.run('./policy_sentry/bin/cli.py query condition-table --service cloud9 --name cloud9:Permissions', pty=True) + c.run( + "./policy_sentry/bin/cli.py query condition-table --service cloud9", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query condition-table --service cloud9 --name cloud9:Permissions", + pty=True, + ) except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -180,17 +223,44 @@ def query_with_yaml(c): try: c.run('echo "Querying the action table with yaml option"') c.run('echo "Querying the action table"', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ram --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ram --name tagresource --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ram --access-level permissions-management --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query action-table --service ses --condition ses:FeedbackAddress --fmt yaml', pty=True) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ram --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ram --name tagresource --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ram --access-level permissions-management --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query action-table --service ses --condition ses:FeedbackAddress --fmt yaml", + pty=True, + ) c.run('echo "Querying the ARN table"', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service ssm --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service cloud9 --name environment --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query arn-table --service cloud9 --list-arn-types --fmt yaml', pty=True) + c.run( + "./policy_sentry/bin/cli.py query arn-table --service ssm --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query arn-table --service cloud9 --name environment --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query arn-table --service cloud9 --list-arn-types --fmt yaml", + pty=True, + ) c.run('echo "Querying the condition keys table"', pty=True) - c.run('./policy_sentry/bin/cli.py query condition-table --service cloud9 --fmt yaml', pty=True) - c.run('./policy_sentry/bin/cli.py query condition-table --service cloud9 --name cloud9:Permissions --fmt yaml', pty=True) + c.run( + "./policy_sentry/bin/cli.py query condition-table --service cloud9 --fmt yaml", + pty=True, + ) + c.run( + "./policy_sentry/bin/cli.py query condition-table --service cloud9 --name cloud9:Permissions --fmt yaml", + pty=True, + ) except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -202,24 +272,9 @@ def query_with_yaml(c): # TEST - SECURITY @task def security_scan(c): - """Runs `bandit` and `safety check`""" - try: - c.run('bandit -r policy_sentry/') - c.run('safety check') - except UnexpectedExit as u_e: - logger.critical(f"FAIL! UnexpectedExit: {u_e}") - sys.exit(1) - except Failure as f_e: - logger.critical(f"FAIL: Failure: {f_e}") - sys.exit(1) - - -# TEST - LINT -@task -def run_linter(c): - """Linting with `pylint`""" + """Runs `safety check`""" try: - c.run('pylint policy_sentry/', warn=False) + c.run("safety check") except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -233,7 +288,7 @@ def run_linter(c): def run_mypy(c): """Type checking with `mypy`""" try: - c.run('mypy') + c.run("mypy") except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -248,8 +303,8 @@ def run_pytest(c): """Unit testing: Runs unit tests using `pytest`""" c.run('echo "Running Unit tests"') try: - c.run('python -m coverage run -m pytest -v') - c.run('python -m coverage report -m') + c.run("python -m coverage run -m pytest -v") + c.run("python -m coverage report -m") except UnexpectedExit as u_e: logger.critical(f"FAIL! UnexpectedExit: {u_e}") sys.exit(1) @@ -262,33 +317,32 @@ def run_pytest(c): @task def build_docker(c): """Open HTML docs in Google Chrome locally on your computer""" - c.run('docker build -t kmcquade/policy_sentry .') + c.run("docker build -t kmcquade/policy_sentry .") # Add all testing tasks to the test collection -integration.add_task(clean_config_directory, 'clean') -integration.add_task(version_check, 'version') -integration.add_task(create_db, 'initialize') -integration.add_task(write_policy, 'write-policy') -integration.add_task(query, 'query') -integration.add_task(query_with_yaml, 'query-yaml') +integration.add_task(clean_config_directory, "clean") +integration.add_task(version_check, "version") +integration.add_task(create_db, "initialize") +integration.add_task(write_policy, "write-policy") +integration.add_task(query, "query") +integration.add_task(query_with_yaml, "query-yaml") -unit.add_task(run_pytest, 'pytest') +unit.add_task(run_pytest, "pytest") docs.add_task(build_docs, "build-docs") docs.add_task(serve_docs, "serve-docs") -docs.add_task(download_latest_aws_docs, 'download_latest_aws_docs') +docs.add_task(download_latest_aws_docs, "download_latest_aws_docs") # test.add_task(run_full_test_suite, 'all') -test.add_task(run_linter, 'lint') -test.add_task(run_mypy, 'type-check') -test.add_task(security_scan, 'security') - -build.add_task(build_package, 'build-package') -build.add_task(install_package, 'install-package') -build.add_task(uninstall_package, 'uninstall-package') -build.add_task(upload_to_pypi_test_server, 'upload-test') -build.add_task(upload_to_pypi_prod_server, 'upload-prod') -build.add_task(upload_to_pypi_prod_server, 'upload-prod') - -docker.add_task(build_docker, 'build-docker') +test.add_task(run_mypy, "type-check") +test.add_task(security_scan, "security") + +build.add_task(build_package, "build-package") +build.add_task(install_package, "install-package") +build.add_task(uninstall_package, "uninstall-package") +build.add_task(upload_to_pypi_test_server, "upload-test") +build.add_task(upload_to_pypi_prod_server, "upload-prod") +build.add_task(upload_to_pypi_prod_server, "upload-prod") + +docker.add_task(build_docker, "build-docker")