diff --git a/.github/scripts/update_data.py b/.github/scripts/update_data.py
index a2e4efcb3..f3dd5334d 100644
--- a/.github/scripts/update_data.py
+++ b/.github/scripts/update_data.py
@@ -7,5 +7,5 @@
from policy_sentry.command.initialize import initialize
-if __name__ == '__main__':
+if __name__ == "__main__":
initialize(None, True, True)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4c4766f19..5e87d9f45 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/rhysd/actionlint
- rev: v1.6.26
+ rev: v1.7.1
hooks:
- id: actionlint-docker
- repo: https://github.com/antonbabenko/pre-commit-terraform
@@ -10,12 +10,9 @@ repos:
# - id: terraform_docs
# args: ['--sort-by-required', '--no-providers']
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.4.6
+ rev: v0.5.1
hooks:
+ - id: ruff
+ files: ^policy_sentry/
- id: ruff-format
files: ^policy_sentry/
- - repo: https://github.com/asottile/pyupgrade
- rev: v3.11.0
- hooks:
- - id: pyupgrade
- args: ["--py37-plus"]
diff --git a/.pylintrc b/.pylintrc
deleted file mode 100644
index d41375f23..000000000
--- a/.pylintrc
+++ /dev/null
@@ -1,525 +0,0 @@
-[MASTER]
-
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code.
-extension-pkg-whitelist=
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
-
-# Add files or directories matching the regex patterns to the blacklist. The
-# regex matches against base names, not paths.
-ignore-patterns=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
-# number of processors available to use.
-jobs=1
-
-# Control the amount of potential inferred values when inferring a single
-# object. This can help the performance when dealing with large functions or
-# complex, nested conditions.
-limit-inference-results=100
-
-# List of plugins (as comma separated values of python module names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# Specify a configuration file.
-#rcfile=
-
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
-# Allow loading of arbitrary C extensions. Extensions are imported into the
-# active Python interpreter and may run arbitrary code.
-unsafe-load-any-extension=no
-
-
-[MESSAGES CONTROL]
-
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
-confidence=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once). You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use "--disable=all --enable=classes
-# --disable=W".
-disable=raw-checker-failed,
- bad-inline-option,
- locally-disabled,
- file-ignored,
- suppressed-message,
- useless-suppression,
- deprecated-pragma,
- use-symbolic-message-instead,
- fixme,
- line-too-long,
- duplicate-code,
- consider-using-sys-exit,
- no-else-return,
- too-few-public-methods,
- too-many-nested-blocks,
- too-many-statements,
- too-many-branches,
- self-assigning-variable,
- pointless-string-statement,
- too-many-locals,
- consider-using-enumerate,
- too-many-arguments,
- expression-not-assigned,
- invalid-name,
- logging-too-many-args,
- logging-format-interpolation,
- f-string-without-interpolation,
- logging-fstring-interpolation,
- unused-variable,
- broad-exception-raised,
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
-
-
-[REPORTS]
-
-# Python expression which should return a score less than or equal to 10. You
-# have access to the variables 'error', 'warning', 'refactor', and 'convention'
-# which contain the number of messages in each category, as well as 'statement'
-# which is the total number of statements analyzed. This score is used by the
-# global evaluation report (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details.
-#msg-template=
-
-# Set the output format. Available formats are text, parseable, colorized, json
-# and msvs (visual studio). You can also give a reporter class, e.g.
-# mypackage.mymodule.MyReporterClass.
-output-format=text
-
-# Tells whether to display a full report or only the messages.
-reports=no
-
-# Activate the evaluation score.
-score=yes
-
-
-[REFACTORING]
-
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
-
-# Complete name of functions that never returns. When checking for
-# inconsistent-return-statements if a never returning function is called then
-# it will be considered as an explicit return statement and no message will be
-# printed.
-never-returning-functions=sys.exit
-
-
-[LOGGING]
-
-# Format style used to check logging format string. `old` means using %
-# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings.
-logging-format-style=old
-
-# Logging modules to check that the string format arguments are in logging
-# function parameter format.
-logging-modules=logging
-
-
-[SPELLING]
-
-# Limits count of emitted suggestions for spelling mistakes.
-max-spelling-suggestions=4
-
-# Spelling dictionary name. Available dictionaries: none. To make it work,
-# install the python-enchant package.
-spelling-dict=
-
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
-
-# A path to a file that contains the private dictionary; one word per line.
-spelling-private-dict-file=
-
-# Tells whether to store unknown words to the private dictionary (see the
-# --spelling-private-dict-file option) instead of raising a message.
-spelling-store-unknown-words=no
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,
- XXX,
- TODO
-
-
-[TYPECHECK]
-
-# List of decorators that produce context managers, such as
-# contextlib.contextmanager. Add to this list to register other decorators that
-# produce valid context managers.
-contextmanager-decorators=contextlib.contextmanager
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# Tells whether to warn about missing members when the owner of the attribute
-# is inferred to be None.
-ignore-none=yes
-
-# This flag controls whether pylint should warn about no-member and similar
-# checks whenever an opaque object is returned when inferring. The inference
-# can return multiple potential results while evaluating a Python object, but
-# some branches might not be evaluated, which results in partial inference. In
-# that case, it might be useful to still emit no-member and other checks for
-# the rest of the inferred objects.
-ignore-on-opaque-inference=yes
-
-# List of class names for which member attributes should not be checked (useful
-# for classes with dynamically set attributes). This supports the use of
-# qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis). It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
-
-# Show a hint with possible names when a member name was not found. The aspect
-# of finding the hint is based on edit distance.
-missing-member-hint=yes
-
-# The minimum edit distance a name should have in order to be considered a
-# similar match for a missing member name.
-missing-member-hint-distance=1
-
-# The total number of similar names that should be taken in consideration when
-# showing a hint for a missing member.
-missing-member-max-choices=1
-
-# List of decorators that change the signature of a decorated function.
-signature-mutators=
-
-
-[VARIABLES]
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid defining new builtins when possible.
-additional-builtins=
-
-# Tells whether unused global variables should be treated as a violation.
-allow-global-unused-variables=yes
-
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,
- _cb
-
-# A regular expression matching the name of dummy variables (i.e. expected to
-# not be used).
-dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# List of qualified module names which can have objects that can redefine
-# builtins.
-redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
-
-
-[FORMAT]
-
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
-
-# Regexp for a line that is allowed to be longer than the limit.
-ignore-long-lines=^\s*(# )??$
-
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
-
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string=' '
-
-# Maximum number of characters on a single line.
-max-line-length=119
-
-# Maximum number of lines in a module.
-max-module-lines=1000
-
-# Allow the body of a class to be on the same line as the declaration if body
-# contains single statement.
-single-line-class-stmt=no
-
-# Allow the body of an if to be on the same line as the test if there is no
-# else.
-single-line-if-stmt=no
-
-
-[SIMILARITIES]
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-# Ignore imports when computing similarities.
-ignore-imports=no
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-
-[BASIC]
-
-# Naming style matching correct argument names.
-argument-naming-style=snake_case
-
-# Regular expression matching correct argument names. Overrides argument-
-# naming-style.
-#argument-rgx=
-
-# Naming style matching correct attribute names.
-attr-naming-style=snake_case
-
-# Regular expression matching correct attribute names. Overrides attr-naming-
-# style.
-#attr-rgx=
-
-# Bad variable names which should always be refused, separated by a comma.
-bad-names=foo,
- bar,
- baz,
- toto,
- tutu,
- tata
-
-# Naming style matching correct class attribute names.
-class-attribute-naming-style=any
-
-# Regular expression matching correct class attribute names. Overrides class-
-# attribute-naming-style.
-#class-attribute-rgx=
-
-# Naming style matching correct class names.
-class-naming-style=PascalCase
-
-# Regular expression matching correct class names. Overrides class-naming-
-# style.
-#class-rgx=
-
-# Naming style matching correct constant names.
-const-naming-style=UPPER_CASE
-
-# Regular expression matching correct constant names. Overrides const-naming-
-# style.
-#const-rgx=
-
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
-
-# Naming style matching correct function names.
-function-naming-style=snake_case
-
-# Regular expression matching correct function names. Overrides function-
-# naming-style.
-#function-rgx=
-
-# Good variable names which should always be accepted, separated by a comma.
-good-names=i,
- j,
- k,
- ex,
- Run,
- _
-
-# Include a hint for the correct naming format with invalid-name.
-include-naming-hint=no
-
-# Naming style matching correct inline iteration names.
-inlinevar-naming-style=any
-
-# Regular expression matching correct inline iteration names. Overrides
-# inlinevar-naming-style.
-#inlinevar-rgx=
-
-# Naming style matching correct method names.
-method-naming-style=snake_case
-
-# Regular expression matching correct method names. Overrides method-naming-
-# style.
-#method-rgx=
-
-# Naming style matching correct module names.
-module-naming-style=snake_case
-
-# Regular expression matching correct module names. Overrides module-naming-
-# style.
-#module-rgx=
-
-# Colon-delimited sets of names that determine each other's naming style when
-# the name regexes allow several styles.
-name-group=
-
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=^_
-
-# List of decorators that produce properties, such as abc.abstractproperty. Add
-# to this list to register other decorators that produce valid properties.
-# These decorators are taken in consideration only for invalid-name.
-property-classes=abc.abstractproperty
-
-# Naming style matching correct variable names.
-variable-naming-style=snake_case
-
-# Regular expression matching correct variable names. Overrides variable-
-# naming-style.
-#variable-rgx=
-
-
-[STRING]
-
-# This flag controls whether the implicit-str-concat-in-sequence should
-# generate a warning on implicit string concatenation in sequences defined over
-# several lines.
-check-str-concat-over-line-jumps=no
-
-
-[IMPORTS]
-
-# List of modules that can be imported at any level, not just the top level
-# one.
-allow-any-import-level=
-
-# Allow wildcard imports from modules that define __all__.
-allow-wildcard-with-all=no
-
-# Analyse import fallback blocks. This can be used to support both Python 2 and
-# 3 compatible code, which means that the block might have code that exists
-# only in one or another interpreter, leading to false positives when analysed.
-analyse-fallback-blocks=no
-
-# Deprecated modules which should not be used, separated by a comma.
-deprecated-modules=optparse,tkinter.tix
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled).
-ext-import-graph=
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled).
-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled).
-int-import-graph=
-
-# Force import order to recognize a module as part of the standard
-# compatibility libraries.
-known-standard-library=
-
-# Force import order to recognize a module as part of a third party library.
-known-third-party=enchant
-
-# Couples of modules and preferred modules, separated by a comma.
-preferred-modules=
-
-
-[CLASSES]
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,
- __new__,
- setUp,
- __post_init__
-
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,
- _fields,
- _replace,
- _source,
- _make
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=cls
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method.
-max-args=5
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
-
-# Maximum number of boolean expressions in an if statement (see R0916).
-max-bool-expr=5
-
-# Maximum number of branch for function / method body.
-max-branches=12
-
-# Maximum number of locals for function / method body.
-max-locals=15
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
-
-# Maximum number of return / yield for function / method body.
-max-returns=6
-
-# Maximum number of statements in function / method body.
-max-statements=50
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "BaseException, Exception".
-overgeneral-exceptions=builtins.BaseException,
- builtins.Exception
diff --git a/Makefile b/Makefile
index c38edfbf9..6cc7a87cf 100644
--- a/Makefile
+++ b/Makefile
@@ -50,15 +50,6 @@ clean:
# ---------------------------------------------------------------------------------------------------------------------
test: setup-dev
python3 -m coverage run -m pytest -v
-security-test: setup-dev
- bandit -r ./${PROJECT_UNDERSCORE}/
-# ---------------------------------------------------------------------------------------------------------------------
-# Linting and formatting
-# ---------------------------------------------------------------------------------------------------------------------
-fmt: setup-dev
- black ${PROJECT_UNDERSCORE}/
-lint: setup-dev
- pylint ${PROJECT_UNDERSCORE}/
# ---------------------------------------------------------------------------------------------------------------------
# Package publishing
# ---------------------------------------------------------------------------------------------------------------------
diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md
index f13158119..ddc0d8cef 100644
--- a/docs/contributing/testing.md
+++ b/docs/contributing/testing.md
@@ -48,8 +48,7 @@ Available tasks:
integration.version Print the version
integration.write-policy Integration testing: Tests the `write-policy`
function.
- test.lint Linting with `pylint` and `autopep8`
- test.security Runs `bandit` and `safety check`
+ test.security Runs `safety check`
unit.pytest Unit testing: Runs unit tests using `pytest`
diff --git a/policy_sentry/analysis/expand.py b/policy_sentry/analysis/expand.py
index 317806184..86d5cb8e7 100644
--- a/policy_sentry/analysis/expand.py
+++ b/policy_sentry/analysis/expand.py
@@ -2,9 +2,9 @@
from __future__ import annotations
-import logging
import copy
import fnmatch
+import logging
from typing import Any
from policy_sentry.querying.actions import get_actions_for_service
diff --git a/policy_sentry/bin/cli.py b/policy_sentry/bin/cli.py
index b9add3b05..aa7f46085 100755
--- a/policy_sentry/bin/cli.py
+++ b/policy_sentry/bin/cli.py
@@ -4,6 +4,7 @@
"""
import click
+
from policy_sentry import command
from policy_sentry.bin.version import __version__
diff --git a/policy_sentry/command/__init__.py b/policy_sentry/command/__init__.py
index b6ca93147..8f200ae79 100644
--- a/policy_sentry/command/__init__.py
+++ b/policy_sentry/command/__init__.py
@@ -1,5 +1,2 @@
-# pylint: disable=missing-module-docstring
-from policy_sentry.command import initialize
-from policy_sentry.command import write_policy
-from policy_sentry.command import create_template
-from policy_sentry.command import query
+# ruff: noqa: F401
+from policy_sentry.command import create_template, initialize, query, write_policy
diff --git a/policy_sentry/command/create_template.py b/policy_sentry/command/create_template.py
index 73bb6083c..d145c05ad 100644
--- a/policy_sentry/command/create_template.py
+++ b/policy_sentry/command/create_template.py
@@ -5,11 +5,13 @@
from __future__ import annotations
-from pathlib import Path
import logging
+from pathlib import Path
+
import click
-from policy_sentry.writing.template import create_actions_template, create_crud_template
+
from policy_sentry import set_stream_logger
+from policy_sentry.writing.template import create_actions_template, create_crud_template
logger = logging.getLogger(__name__)
diff --git a/policy_sentry/command/initialize.py b/policy_sentry/command/initialize.py
index 00729f030..161ad2433 100755
--- a/policy_sentry/command/initialize.py
+++ b/policy_sentry/command/initialize.py
@@ -5,26 +5,28 @@
from __future__ import annotations
+import logging
import os
import shutil
-import logging
+
import click
+
+from policy_sentry import set_stream_logger
from policy_sentry.querying.all import get_all_service_prefixes
from policy_sentry.shared.awsdocs import (
- update_html_docs_directory,
create_database,
+ update_html_docs_directory,
)
from policy_sentry.shared.constants import (
- LOCAL_HTML_DIRECTORY_PATH,
+ BUNDLED_DATA_DIRECTORY,
+ BUNDLED_DATASTORE_FILE_PATH,
+ BUNDLED_HTML_DIRECTORY_PATH,
CONFIG_DIRECTORY,
- LOCAL_DATASTORE_FILE_PATH,
DATASTORE_FILE_PATH,
LOCAL_ACCESS_OVERRIDES_FILE,
- BUNDLED_HTML_DIRECTORY_PATH,
- BUNDLED_DATASTORE_FILE_PATH,
- BUNDLED_DATA_DIRECTORY,
+ LOCAL_DATASTORE_FILE_PATH,
+ LOCAL_HTML_DIRECTORY_PATH,
)
-from policy_sentry import set_stream_logger
logger = logging.getLogger(__name__)
diff --git a/policy_sentry/command/query.py b/policy_sentry/command/query.py
index 76a7f0928..521dc8a4d 100644
--- a/policy_sentry/command/query.py
+++ b/policy_sentry/command/query.py
@@ -4,51 +4,51 @@
from __future__ import annotations
-import os
import json
import logging
+import os
from typing import Any
import click
import yaml
-from policy_sentry.querying.services import get_services_data
-from policy_sentry.util.access_levels import transform_access_level_text
+from policy_sentry import set_stream_logger
+from policy_sentry.querying.actions import (
+ get_action_data,
+ get_actions_for_service,
+ get_actions_matching_arn_type,
+ get_actions_matching_condition_key,
+ get_actions_that_support_wildcard_arns_only,
+ get_actions_with_access_level,
+ get_actions_with_arn_type_and_access_level,
+)
from policy_sentry.querying.all import get_all_service_prefixes
from policy_sentry.querying.arns import (
get_arn_type_details,
get_arn_types_for_service,
get_raw_arns_for_service,
)
-from policy_sentry.querying.actions import (
- get_actions_for_service,
- get_actions_with_access_level,
- get_action_data,
- get_actions_matching_condition_key,
- get_actions_with_arn_type_and_access_level,
- get_actions_matching_arn_type,
- get_actions_that_support_wildcard_arns_only,
-)
from policy_sentry.querying.conditions import (
- get_condition_keys_for_service,
get_condition_key_details,
+ get_condition_keys_for_service,
)
+from policy_sentry.querying.services import get_services_data
from policy_sentry.shared.constants import (
DATASTORE_FILE_PATH,
LOCAL_DATASTORE_FILE_PATH,
)
-from policy_sentry import set_stream_logger
+from policy_sentry.util.access_levels import transform_access_level_text
logger = logging.getLogger(__name__)
iam_definition_path = DATASTORE_FILE_PATH
-def print_list(output: Any, fmt: str = "json") -> None:
+def print_list(output: list[Any], fmt: str = "json") -> None:
"""Common method on how to print a list, depending on whether the user requests JSON or YAML output"""
print(yaml.dump(output)) if fmt == "yaml" else [print(item) for item in output]
-def print_dict(output: Any, fmt: str = "json") -> None:
+def print_dict(output: list[Any] | dict[Any, Any], fmt: str = "json") -> None:
"""Common method on how to print a dict, depending on whether the user requests JSON, YAML or CSV output"""
if fmt == "csv":
if not output:
diff --git a/policy_sentry/command/write_policy.py b/policy_sentry/command/write_policy.py
index 7ff27bbde..a1fc0f00e 100755
--- a/policy_sentry/command/write_policy.py
+++ b/policy_sentry/command/write_policy.py
@@ -4,9 +4,9 @@
from __future__ import annotations
-import sys
import json
import logging
+import sys
from pathlib import Path
from typing import Any
@@ -14,9 +14,9 @@
import yaml
from click import Context
+from policy_sentry import set_stream_logger
from policy_sentry.util.file import read_yaml_file
from policy_sentry.writing.sid_group import SidGroup
-from policy_sentry import set_stream_logger
logger = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ class RegisterMinimizeLengthCommand(click.Command):
def parse_args(self, ctx: Context, args: list[str]) -> list[str]:
options = [o for o in ctx.command.params if getattr(o, "register_length", None)]
- prefixes = {p for p in sum([o.opts for o in options], []) if p.startswith("--")}
+ prefixes = {p for p in sum([o.opts for o in options], []) if p.startswith("--")} # noqa: RUF017
for i, a in enumerate(args):
a_tuple = a.split("=")
if a_tuple[0] in prefixes:
diff --git a/policy_sentry/querying/actions.py b/policy_sentry/querying/actions.py
index eddacb669..6da801737 100644
--- a/policy_sentry/querying/actions.py
+++ b/policy_sentry/querying/actions.py
@@ -5,29 +5,29 @@
from __future__ import annotations
-import logging
import functools
+import logging
from typing import Any
from policy_sentry.querying.actions_v1 import (
get_action_data_v1,
get_action_matching_access_level_v1,
- get_actions_with_arn_type_and_access_level_v1,
+ get_actions_for_service_v1,
get_actions_matching_arn_type_v1,
get_actions_matching_arn_v1,
- get_actions_for_service_v1,
-)
-from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2
-from policy_sentry.shared.iam_data import (
- iam_definition,
- get_service_prefix_data,
- get_iam_definition_schema_version,
+ get_actions_with_arn_type_and_access_level_v1,
)
-from policy_sentry.querying.all import get_all_service_prefixes, get_all_actions
+from policy_sentry.querying.all import get_all_actions, get_all_service_prefixes
from policy_sentry.querying.arns import (
get_matching_raw_arns,
get_resource_type_name_with_raw_arn,
)
+from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2
+from policy_sentry.shared.iam_data import (
+ get_iam_definition_schema_version,
+ get_service_prefix_data,
+ iam_definition,
+)
from policy_sentry.util.arns import get_service_from_arn
all_service_prefixes = get_all_service_prefixes()
@@ -320,12 +320,12 @@ def get_actions_with_arn_type_and_access_level_v2(
else:
service_prefix_data = get_service_prefix_data(service_prefix)
for action_name, action_data in service_prefix_data["privileges"].items():
- if action_data["access_level"] == access_level:
- if (
- resource_type_name.lower()
- in action_data["resource_types_lower_name"]
- ):
- results.append(f"{service_prefix}:{action_name}")
+ if (
+ action_data["access_level"] == access_level
+ and resource_type_name.lower()
+ in action_data["resource_types_lower_name"]
+ ):
+ results.append(f"{service_prefix}:{action_name}")
return results
@@ -352,9 +352,10 @@ def get_actions_that_support_wildcard_arns_only(service_prefix: str) -> list[str
else:
service_prefix_data = get_service_prefix_data(service_prefix)
for action_name, action_data in service_prefix_data["privileges"].items():
- if len(action_data["resource_types"]) == 1:
- if action_data["resource_types"].get(""):
- results.append(f"{service_prefix}:{action_name}")
+ if len(action_data["resource_types"]) == 1 and action_data[
+ "resource_types"
+ ].get(""):
+ results.append(f"{service_prefix}:{action_name}")
return results
@@ -469,8 +470,8 @@ def get_actions_matching_condition_key(
Returns:
List: A list of actions
"""
- results = []
if service_prefix == "all":
+ results = []
for some_prefix in all_service_prefixes:
actions = get_actions_matching_condition_key(
service_prefix=some_prefix,
@@ -480,10 +481,12 @@ def get_actions_matching_condition_key(
results.extend(actions)
else:
service_prefix_data = get_service_prefix_data(service_prefix)
- for action_name, action_data in service_prefix_data["privileges"].items():
- for resource_data in action_data["resource_types"].values():
- if condition_key in resource_data["condition_keys"]:
- results.append(f"{service_prefix}:{action_name}")
+ results = [
+ f"{service_prefix}:{action_name}"
+ for action_name, action_data in service_prefix_data["privileges"].items()
+ for resource_data in action_data["resource_types"].values()
+ if condition_key in resource_data["condition_keys"]
+ ]
return results
@@ -637,10 +640,12 @@ def remove_actions_that_are_not_wildcard_arn_only(actions_list: list[str]) -> li
for action in actions_list_unique:
service_prefix, action_name = action.split(":")
action_data = get_action_data(service_prefix, action_name)
- if len(action_data[service_prefix]) == 1:
- if action_data[service_prefix][0]["resource_arn_format"] == "*":
- # Let's return the CamelCase action name format
- results.append(action_data[service_prefix][0]["action"])
+ if (
+ len(action_data[service_prefix]) == 1
+ and action_data[service_prefix][0]["resource_arn_format"] == "*"
+ ):
+ # Let's return the CamelCase action name format
+ results.append(action_data[service_prefix][0]["action"])
return results
diff --git a/policy_sentry/querying/actions_v1.py b/policy_sentry/querying/actions_v1.py
index 691ecf8aa..b8ca3f229 100644
--- a/policy_sentry/querying/actions_v1.py
+++ b/policy_sentry/querying/actions_v1.py
@@ -29,7 +29,9 @@ def get_actions_for_service_v1(
Returns:
List: A list of actions
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
service_prefix_data = get_service_prefix_data(service_prefix)
results = []
@@ -62,7 +64,9 @@ def get_action_data_v1(
Returns:
List: A dictionary containing metadata about an IAM Action.
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
results = []
action_data_results = {}
@@ -74,17 +78,17 @@ def get_action_data_v1(
# Get the baseline conditions and dependent actions
condition_keys = []
dependent_actions = []
- rows = []
if action_name == "*":
- # rows = this_action_data["resource_types"]
- for resource_type_entry in this_action_data["resource_types"]:
- rows.append(this_action_data["resource_types"][resource_type_entry])
+ rows = [
+ this_action_data["resource_types"][resource_type_entry]
+ for resource_type_entry in this_action_data["resource_types"]
+ ]
else:
- for resource_type_entry in this_action_data["resource_types"]:
- if this_action_name.lower() == action_name.lower():
- rows.append(
- this_action_data["resource_types"][resource_type_entry]
- )
+ rows = [
+ this_action_data["resource_types"][resource_type_entry]
+ for resource_type_entry in this_action_data["resource_types"]
+ if this_action_name.lower() == action_name.lower()
+ ]
for row in rows:
# Set default value for if no other matches are found
resource_arn_format = "*"
@@ -92,9 +96,7 @@ def get_action_data_v1(
if row["dependent_actions"]:
dependent_actions.extend(row["dependent_actions"])
# Get the condition keys
- for service_resource_name, service_resource_data in service_prefix_data[
- "resources"
- ].items():
+ for service_resource_data in service_prefix_data["resources"].values():
if row["resource_type"] == "":
continue
if (
@@ -138,7 +140,9 @@ def get_action_matching_access_level_v1(
Returns:
List: action or None
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
result = None
service_prefix_data = get_service_prefix_data(service_prefix.lower())
@@ -146,10 +150,12 @@ def get_action_matching_access_level_v1(
privileges = service_prefix_data.get("privileges")
if privileges:
for this_action_name, action_data in privileges.items():
- if action_data["access_level"] == access_level:
- if this_action_name.lower() == action_name.lower():
- result = f"{service_prefix}:{this_action_name}"
- break
+ if (
+ action_data["access_level"] == access_level
+ and this_action_name.lower() == action_name.lower()
+ ):
+ result = f"{service_prefix}:{this_action_name}"
+ break
return result
@@ -169,7 +175,9 @@ def get_actions_with_arn_type_and_access_level_v1(
Return:
List: A list of actions that have that ARN type and Access level
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
service_prefix_data = get_service_prefix_data(service_prefix)
results = []
@@ -177,11 +185,9 @@ def get_actions_with_arn_type_and_access_level_v1(
if service_prefix == "all":
for some_prefix in all_service_prefixes:
service_prefix_data = get_service_prefix_data(some_prefix)
- for action_name, action_data in service_prefix_data["privileges"].items():
+ for action_data in service_prefix_data["privileges"].values():
if action_data["access_level"] == access_level:
- for resource_name, resource_data in action_data[
- "resource_types"
- ].items():
+ for resource_data in action_data["resource_types"].values():
this_resource_type = resource_data["resource_type"].strip("*")
if this_resource_type.lower() == resource_type_name.lower():
results.append(
@@ -189,11 +195,9 @@ def get_actions_with_arn_type_and_access_level_v1(
)
break
else:
- for action_name, action_data in service_prefix_data["privileges"].items():
+ for action_data in service_prefix_data["privileges"].values():
if action_data["access_level"] == access_level:
- for resource_name, resource_data in action_data[
- "resource_types"
- ].items():
+ for resource_data in action_data["resource_types"].values():
this_resource_type = resource_data["resource_type"].strip("*")
if this_resource_type.lower() == resource_type_name.lower():
results.append(f"{service_prefix}:{action_data['privilege']}")
@@ -215,7 +219,9 @@ def get_actions_matching_arn_type_v1(
Return:
List: A list of actions that have that ARN type
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
service_prefix_data = get_service_prefix_data(service_prefix)
results = []
@@ -223,17 +229,15 @@ def get_actions_matching_arn_type_v1(
if service_prefix == "all":
for some_prefix in all_service_prefixes:
service_prefix_data = get_service_prefix_data(some_prefix)
- for action_name, action_data in service_prefix_data["privileges"].items():
- for resource_name, resource_data in action_data[
- "resource_types"
- ].items():
+ for action_data in service_prefix_data["privileges"].values():
+ for resource_data in action_data["resource_types"].values():
this_resource_type = resource_data["resource_type"].strip("*")
if this_resource_type.lower() == resource_type_name.lower():
results.append(f"{service_prefix}:{action_data['privilege']}")
break
else:
- for action_name, action_data in service_prefix_data["privileges"].items():
- for resource_name, resource_data in action_data["resource_types"].items():
+ for action_data in service_prefix_data["privileges"].values():
+ for resource_data in action_data["resource_types"].values():
this_resource_type = resource_data["resource_type"].strip("*")
if this_resource_type.lower() == resource_type_name.lower():
results.append(f"{service_prefix}:{action_data['privilege']}")
@@ -252,7 +256,9 @@ def get_actions_matching_arn_v1(arn: str) -> list[str]:
Returns:
List: A list of all actions that can match it.
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
raw_arns = get_matching_raw_arns(arn)
results = []
@@ -263,9 +269,9 @@ def get_actions_matching_arn_v1(arn: str) -> list[str]:
service_prefix = get_service_from_arn(raw_arn)
service_prefix_data = get_service_prefix_data(service_prefix)
- for action_name, action_data in service_prefix_data["privileges"].items():
+ for action_data in service_prefix_data["privileges"].values():
# for some_action in service_prefix_data["privileges"]:
- for resource_name, resource_data in action_data["resource_types"].items():
+ for resource_data in action_data["resource_types"].values():
this_resource_type = resource_data["resource_type"].strip("*")
if this_resource_type.lower() == resource_type_name.lower():
results.append(f"{service_prefix}:{action_data['privilege']}")
diff --git a/policy_sentry/querying/all.py b/policy_sentry/querying/all.py
index 72d6fc7ee..942419c5c 100644
--- a/policy_sentry/querying/all.py
+++ b/policy_sentry/querying/all.py
@@ -2,8 +2,8 @@
from __future__ import annotations
-import logging
import functools
+import logging
from policy_sentry.querying.all_v1 import get_all_actions_v1
from policy_sentry.shared.constants import (
@@ -11,9 +11,9 @@
POLICY_SENTRY_SCHEMA_VERSION_V2,
)
from policy_sentry.shared.iam_data import (
- iam_definition,
- get_service_prefix_data,
get_iam_definition_schema_version,
+ get_service_prefix_data,
+ iam_definition,
)
logger = logging.getLogger(__name__)
@@ -31,8 +31,7 @@ def get_all_service_prefixes() -> set[str]:
List: A list of all AWS service prefixes present in the table.
"""
results = set(iam_definition.keys())
- if POLICY_SENTRY_SCHEMA_VERSION_NAME in results:
- results.remove(POLICY_SENTRY_SCHEMA_VERSION_NAME)
+ results.discard(POLICY_SENTRY_SCHEMA_VERSION_NAME)
return results
diff --git a/policy_sentry/querying/all_v1.py b/policy_sentry/querying/all_v1.py
index 983ec2e42..3b0640338 100644
--- a/policy_sentry/querying/all_v1.py
+++ b/policy_sentry/querying/all_v1.py
@@ -17,7 +17,9 @@ def get_all_actions_v1(
:param lowercase: Set to true to have the list of actions be in all lowercase strings.
:return: A list of all actions present in the database.
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
all_actions = set()
diff --git a/policy_sentry/querying/arns.py b/policy_sentry/querying/arns.py
index 349f9137f..b47d09e5b 100644
--- a/policy_sentry/querying/arns.py
+++ b/policy_sentry/querying/arns.py
@@ -5,16 +5,16 @@
from __future__ import annotations
-import logging
import functools
+import logging
import warnings
from typing import Any, cast
from policy_sentry.querying.arns_v1 import get_arn_type_details_v1
from policy_sentry.shared.constants import POLICY_SENTRY_SCHEMA_VERSION_V2
from policy_sentry.shared.iam_data import (
- get_service_prefix_data,
get_iam_definition_schema_version,
+ get_service_prefix_data,
)
from policy_sentry.util.arns import does_arn_match, get_service_from_arn
@@ -33,11 +33,13 @@ def get_arn_data(service_prefix: str, resource_type_name: str) -> list[dict[str,
Returns:
Dictionary: Metadata about an ARN type
"""
- warnings.warn("Please use get_arn_type_details() instead", DeprecationWarning)
+ warnings.warn(
+ "Please use get_arn_type_details() instead", DeprecationWarning, stacklevel=2
+ )
results = []
service_prefix_data = get_service_prefix_data(service_prefix)
- for resource_name, resource_data in service_prefix_data["resources"].items():
+ for resource_data in service_prefix_data["resources"].values():
if resource_data["resource"].lower() == resource_type_name.lower():
output = {
"resource_type_name": resource_data["resource"],
diff --git a/policy_sentry/querying/arns_v1.py b/policy_sentry/querying/arns_v1.py
index e8ef2efae..40b1b9e30 100644
--- a/policy_sentry/querying/arns_v1.py
+++ b/policy_sentry/querying/arns_v1.py
@@ -20,11 +20,13 @@ def get_arn_type_details_v1(
Returns:
Dictionary: Metadata about an ARN type
"""
- warnings.warn("Please recreate the IAM datastore file!", DeprecationWarning)
+ warnings.warn(
+ "Please recreate the IAM datastore file!", DeprecationWarning, stacklevel=2
+ )
service_prefix_data = get_service_prefix_data(service_prefix)
output = {}
- for resource_name, resource_data in service_prefix_data["resources"].items():
+ for resource_data in service_prefix_data["resources"].values():
if resource_data["resource"].lower() == resource_type_name.lower():
output = {
"resource_type_name": resource_data["resource"],
diff --git a/policy_sentry/querying/conditions.py b/policy_sentry/querying/conditions.py
index 9e43e4e9e..64e61a3f6 100644
--- a/policy_sentry/querying/conditions.py
+++ b/policy_sentry/querying/conditions.py
@@ -5,13 +5,13 @@
from __future__ import annotations
-import logging
import functools
+import logging
from typing import cast
+from policy_sentry.querying.actions import get_action_data
from policy_sentry.shared.iam_data import get_service_prefix_data
from policy_sentry.util.conditions import is_condition_key_match
-from policy_sentry.querying.actions import get_action_data
logger = logging.getLogger(__name__)
@@ -107,7 +107,7 @@ def get_condition_value_type(condition_key: str) -> str | None:
Returns:
String: type of the condition key, like Bool, Date, String, etc.
"""
- service_prefix, condition_name = condition_key.split(":")
+ service_prefix, _ = condition_key.split(":")
service_prefix_data = get_service_prefix_data(service_prefix)
for condition_key_entry, condition_key_data in service_prefix_data[
diff --git a/policy_sentry/shared/awsdocs.py b/policy_sentry/shared/awsdocs.py
index 9a39739dd..8b10b44b0 100644
--- a/policy_sentry/shared/awsdocs.py
+++ b/policy_sentry/shared/awsdocs.py
@@ -11,22 +11,22 @@
from __future__ import annotations
-import os
+import json
import logging
+import os
import re
-import json
from pathlib import Path
from typing import Any
import requests
-from bs4 import BeautifulSoup, Tag, PageElement
+from bs4 import BeautifulSoup, PageElement, Tag
from policy_sentry.shared.constants import (
BASE_DOCUMENTATION_URL,
BUNDLED_ACCESS_OVERRIDES_FILE,
LOCAL_HTML_DIRECTORY_PATH,
- POLICY_SENTRY_SCHEMA_VERSION_NAME,
POLICY_SENTRY_SCHEMA_VERSION_LATEST,
+ POLICY_SENTRY_SCHEMA_VERSION_NAME,
)
from policy_sentry.util.access_levels import determine_access_level_override
from policy_sentry.util.file import read_yaml_file
@@ -42,9 +42,7 @@ def header_matches(string: str, table: Tag) -> bool:
if string in header:
match_found = True
break
- if not match_found:
- return False
- return True
+ return match_found
def get_links_from_base_actions_resources_conditions_page() -> list[str]:
@@ -121,13 +119,12 @@ def update_html_docs_directory(html_docs_destination: str) -> None:
for script in soup.find_all("script"):
try:
- if "src" in script.attrs:
- if script.get("src").startswith("/"):
- temp = script.attrs["src"]
- script.attrs["src"] = script.attrs["src"].replace(
- temp, f"https://docs.aws.amazon.com{temp}"
- )
- except TypeError as t_e:
+ if "src" in script.attrs and script.get("src").startswith("/"):
+ temp = script.attrs["src"]
+ script.attrs["src"] = script.attrs["src"].replace(
+ temp, f"https://docs.aws.amazon.com{temp}"
+ )
+ except TypeError as t_e: # noqa: PERF203
logger.warning(t_e)
logger.warning(script)
except AttributeError as a_e:
@@ -180,277 +177,276 @@ def create_database(
# for filename in ['list_amazonathena.partial.html']:
file_list = []
for filename in os.listdir(LOCAL_HTML_DIRECTORY_PATH):
- if os.path.isfile(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename)):
- if filename not in file_list:
- file_list.append(filename)
+ if (
+ os.path.isfile(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename))
+ and filename not in file_list
+ ):
+ file_list.append(filename)
file_list.sort()
for filename in file_list:
if not filename.startswith("list_"):
continue
- with open(os.path.join(LOCAL_HTML_DIRECTORY_PATH, filename)) as f:
- soup = BeautifulSoup(f.read(), "html.parser")
- main_content = soup.find(id="main-content")
- if not isinstance(main_content, Tag):
- continue
+ content = (Path(LOCAL_HTML_DIRECTORY_PATH) / filename).read_text()
+ soup = BeautifulSoup(content, "html.parser")
+ main_content = soup.find(id="main-content")
+ if not isinstance(main_content, Tag):
+ continue
- # Get service name
- topic_title = main_content.find("h1", class_="topictitle")
- if not isinstance(topic_title, PageElement):
- continue
+ # Get service name
+ topic_title = main_content.find("h1", class_="topictitle")
+ if not isinstance(topic_title, PageElement):
+ continue
- title = re.sub(
- ".*Actions, resources, and condition Keys for *",
- "",
- topic_title.text,
- flags=re.IGNORECASE,
- )
- title = title.replace("", "")
- service_name = chomp(title)
+ title = re.sub(
+ ".*Actions, resources, and condition Keys for *",
+ "",
+ topic_title.text,
+ flags=re.IGNORECASE,
+ )
+ title = title.replace("", "")
+ service_name = chomp(title)
- service_prefix = ""
- title_parent = topic_title.parent
- if title_parent is None:
- continue
+ service_prefix = ""
+ title_parent = topic_title.parent
+ if title_parent is None:
+ continue
- for c in title_parent.children:
- if "prefix" in str(c):
- service_prefix = str(c)
- service_prefix = service_prefix.split('')[1]
- service_prefix = chomp(service_prefix.split("
")[0])
- break
+ for c in title_parent.children:
+ if "prefix" in str(c):
+ service_prefix = str(c)
+ service_prefix = service_prefix.split('')[1]
+ service_prefix = chomp(service_prefix.split("
")[0])
+ break
+
+ if service_prefix not in schema:
+ # The URL to that service's Actions, Resources, and Condition Keys page
+ service_authorization_url_prefix = (
+ "https://docs.aws.amazon.com/service-authorization/latest/reference"
+ )
+ service_authorization_url = f"{service_authorization_url_prefix}/{filename}"
+ schema[service_prefix] = {
+ "service_name": service_name,
+ "prefix": service_prefix,
+ "service_authorization_url": service_authorization_url,
+ "privileges": {},
+ "privileges_lower_name": {}, # used for faster lookups
+ "resources": {},
+ "resources_lower_name": {}, # used for faster lookups
+ "conditions": {},
+ }
+
+ access_level_overrides_cfg = get_action_access_level_overrides_from_yml(
+ service_prefix, access_level_overrides_file
+ )
- if service_prefix not in schema:
- # The URL to that service's Actions, Resources, and Condition Keys page
- service_authorization_url_prefix = (
- "https://docs.aws.amazon.com/service-authorization/latest/reference"
- )
- service_authorization_url = (
- f"{service_authorization_url_prefix}/{filename}"
- )
- schema[service_prefix] = {
- "service_name": service_name,
- "prefix": service_prefix,
- "service_authorization_url": service_authorization_url,
- "privileges": {},
- "privileges_lower_name": {}, # used for faster lookups
- "resources": {},
- "resources_lower_name": {}, # used for faster lookups
- "conditions": {},
- }
+ tables = main_content.find_all("div", class_="table-contents")
- access_level_overrides_cfg = get_action_access_level_overrides_from_yml(
- service_prefix, access_level_overrides_file
- )
+ for table in tables:
+ # There can be 3 tables, the actions table, an ARN table, and a condition key table
+ # Example: https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awssecuritytokenservice.html
+ if not header_matches("actions", table) or not header_matches(
+ "description", table
+ ):
+ continue
- tables = main_content.find_all("div", class_="table-contents")
+ rows = table.find_all("tr")
+ row_number = 0
+ while row_number < len(rows):
+ row = rows[row_number]
- for table in tables:
- # There can be 3 tables, the actions table, an ARN table, and a condition key table
- # Example: https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awssecuritytokenservice.html
- if not header_matches("actions", table) or not header_matches(
- "description", table
- ):
+ cells = row.find_all("td")
+ if len(cells) == 0:
+ # Skip the header row, which has th, not td cells
+ row_number += 1
continue
- rows = table.find_all("tr")
- row_number = 0
- while row_number < len(rows):
- row = rows[row_number]
+ if len(cells) != 6:
+ # Sometimes the privilege contains Scenarios, and I don't know how to handle this
+ # raise Exception("Unexpected format in {}: {}".format(prefix, row))
+ break
- cells = row.find_all("td")
- if len(cells) == 0:
- # Skip the header row, which has th, not td cells
- row_number += 1
+ # See if this cell spans multiple rows
+ rowspan = 1
+ if "rowspan" in cells[0].attrs:
+ rowspan = int(cells[0].attrs["rowspan"])
+
+ priv = ""
+ # Get the privilege
+ for link in cells[0].find_all("a"):
+ if "href" not in link.attrs: # pylint: disable=no-else-continue
+ # Skip the tags
+ api_documentation_link = None
continue
-
- if len(cells) != 6:
- # Sometimes the privilege contains Scenarios, and I don't know how to handle this
- # raise Exception("Unexpected format in {}: {}".format(prefix, row))
- break
-
- # See if this cell spans multiple rows
- rowspan = 1
- if "rowspan" in cells[0].attrs:
- rowspan = int(cells[0].attrs["rowspan"])
-
- priv = ""
- # Get the privilege
- for link in cells[0].find_all("a"):
- if "href" not in link.attrs: # pylint: disable=no-else-continue
- # Skip the tags
- api_documentation_link = None
- continue
- else:
- api_documentation_link = link.attrs.get("href")
- logger.debug(api_documentation_link)
- priv = chomp(link.text)
- if priv == "":
- priv = chomp(cells[0].text)
- action_name = priv
- description = chomp(cells[1].text)
- access_level = chomp(cells[2].text)
- # Access Level #####
- # access_level_overrides_cfg will only be true if the service in question is present
- # in the overrides YML file
- if access_level_overrides_cfg:
- override_result = determine_access_level_override(
- service=service_prefix,
- action_name=action_name,
- provided_access_level=access_level,
- service_override_config=access_level_overrides_cfg,
+ else:
+ api_documentation_link = link.attrs.get("href")
+ logger.debug(api_documentation_link)
+ priv = chomp(link.text)
+ if priv == "":
+ priv = chomp(cells[0].text)
+ action_name = priv
+ description = chomp(cells[1].text)
+ access_level = chomp(cells[2].text)
+ # Access Level #####
+ # access_level_overrides_cfg will only be true if the service in question is present
+ # in the overrides YML file
+ if access_level_overrides_cfg:
+ override_result = determine_access_level_override(
+ service=service_prefix,
+ action_name=action_name,
+ provided_access_level=access_level,
+ service_override_config=access_level_overrides_cfg,
+ )
+ if override_result:
+ access_level = override_result
+ logger.debug(
+ "Override: Setting access level for %s:%s to %s",
+ service_prefix,
+ action_name,
+ access_level,
)
- if override_result:
- access_level = override_result
- logger.debug(
- "Override: Setting access level for %s:%s to %s",
- service_prefix,
- action_name,
- access_level,
- )
- # else:
- # access_level = access_level
- # else:
- # access_level = access_level
- resource_types = {}
- resource_types_lower_name = {}
- resource_cell = 3
-
- while rowspan > 0:
- if len(cells) == 3 or len(cells) == 6:
- # ec2:RunInstances contains a few "scenarios" which start in the
- # description field, len(cells) is 5.
- # I'm ignoring these as I don't know how to handle them.
- # These include things like "EC2-Classic-InstanceStore" and
- # "EC2-VPC-InstanceStore-Subnet"
-
- resource_type = chomp(cells[resource_cell].text)
-
- condition_keys_element = cells[resource_cell + 1]
- condition_keys = []
- if condition_keys_element.text != "":
- for key_element in condition_keys_element.find_all("p"):
- condition_keys.append(chomp(key_element.text))
-
- dependent_actions_element = cells[resource_cell + 2]
- dependent_actions = []
- if dependent_actions_element.text != "":
- for (
- action_element
- ) in dependent_actions_element.find_all("p"):
- chomped_action = chomp(action_element.text)
- dependent_actions.append(
- sanitize_service_name(chomped_action)
- )
- if "*" in resource_type:
- required = True
- resource_type = resource_type.strip("*")
- else:
- required = False
-
- resource_types[resource_type] = {
- "resource_type": resource_type,
- "required": required,
- "condition_keys": condition_keys,
- "dependent_actions": dependent_actions,
- }
- resource_types_lower_name[resource_type.lower()] = (
- resource_type
- )
- rowspan -= 1
- if rowspan > 0:
- row_number += 1
- resource_cell = 0
- row = rows[row_number]
- cells = row.find_all("td")
-
- if "[permission only]" in priv:
- priv = priv.split(" ", maxsplit=1)[0]
-
- privilege_schema = {
- "privilege": priv,
- "description": description,
- "access_level": access_level,
- "resource_types": resource_types,
- "resource_types_lower_name": resource_types_lower_name,
- "api_documentation_link": api_documentation_link,
- }
-
- schema[service_prefix]["privileges"][priv] = privilege_schema
- schema[service_prefix]["privileges_lower_name"][priv.lower()] = priv
- row_number += 1
+ # else:
+ # access_level = access_level
+ # else:
+ # access_level = access_level
+ resource_types = {}
+ resource_types_lower_name = {}
+ resource_cell = 3
+
+ while rowspan > 0:
+ if len(cells) == 3 or len(cells) == 6:
+ # ec2:RunInstances contains a few "scenarios" which start in the
+ # description field, len(cells) is 5.
+ # I'm ignoring these as I don't know how to handle them.
+ # These include things like "EC2-Classic-InstanceStore" and
+ # "EC2-VPC-InstanceStore-Subnet"
+
+ resource_type = chomp(cells[resource_cell].text)
+
+ condition_keys_element = cells[resource_cell + 1]
+ condition_keys = [
+ chomp(key_element.text)
+ for key_element in condition_keys_element.find_all("p")
+ if condition_keys_element.text != ""
+ ]
+
+ dependent_actions_element = cells[resource_cell + 2]
+ dependent_actions = []
+ if dependent_actions_element.text != "":
+ for action_element in dependent_actions_element.find_all(
+ "p"
+ ):
+ chomped_action = chomp(action_element.text)
+ dependent_actions.append(
+ sanitize_service_name(chomped_action)
+ )
+ if "*" in resource_type:
+ required = True
+ resource_type = resource_type.strip("*")
+ else:
+ required = False
+
+ resource_types[resource_type] = {
+ "resource_type": resource_type,
+ "required": required,
+ "condition_keys": condition_keys,
+ "dependent_actions": dependent_actions,
+ }
+ resource_types_lower_name[resource_type.lower()] = resource_type
+ rowspan -= 1
+ if rowspan > 0:
+ row_number += 1
+ resource_cell = 0
+ row = rows[row_number]
+ cells = row.find_all("td")
+
+ if "[permission only]" in priv:
+ priv = priv.split(" ", maxsplit=1)[0]
+
+ privilege_schema = {
+ "privilege": priv,
+ "description": description,
+ "access_level": access_level,
+ "resource_types": resource_types,
+ "resource_types_lower_name": resource_types_lower_name,
+ "api_documentation_link": api_documentation_link,
+ }
+
+ schema[service_prefix]["privileges"][priv] = privilege_schema
+ schema[service_prefix]["privileges_lower_name"][priv.lower()] = priv
+ row_number += 1
+
+ # Get resource table
+ for table in tables:
+ if not header_matches("resource types", table) or not header_matches(
+ "arn", table
+ ):
+ continue
+
+ rows = table.find_all("tr")
+ for row in rows:
+ cells = row.find_all("td")
- # Get resource table
- for table in tables:
- if not header_matches("resource types", table) or not header_matches(
- "arn", table
- ):
+ if len(cells) == 0:
+ # Skip the header row, which has th, not td cells
continue
- rows = table.find_all("tr")
- for row in rows:
- cells = row.find_all("td")
+ if len(cells) != 3:
+ raise Exception(
+ f"Unexpected number of resource cells {len(cells)} in {filename}"
+ )
- if len(cells) == 0:
- # Skip the header row, which has th, not td cells
- continue
+ resource = chomp(cells[0].text)
- if len(cells) != 3:
- raise Exception(
- f"Unexpected number of resource cells {len(cells)} in {filename}"
- )
+ arn = no_white_space(cells[1].text)
+ conditions = [
+ chomp(condition.text) for condition in cells[2].find_all("p")
+ ]
- resource = chomp(cells[0].text)
+ schema[service_prefix]["resources"][resource] = {
+ "resource": resource,
+ "arn": arn,
+ "condition_keys": conditions,
+ }
+ schema[service_prefix]["resources_lower_name"][resource.lower()] = (
+ resource
+ )
- arn = no_white_space(cells[1].text)
- conditions = []
- for condition in cells[2].find_all("p"):
- conditions.append(chomp(condition.text))
+ # Get condition keys table
+ for table in tables:
+ if not (
+ header_matches("