Merge pull request #31 from epam/develop

Release 5.5.0, develop to main

.gitignore

@@ -251,4 +251,5 @@ pip-selfcheck.json
 
 .syndicate
 .tmp/
-**/.DS_Store
+**/.DS_Store
+.vscode/

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.5.0] - 2024-08-07
+- return 429 status code if dynamodb provisioned capacity exceeded error
+- remove `accN` index from usage
+- scan each region in a separate process in order to reduce RAM usage
+- change `create_indexes` command. Now it ensures that indexes are up-to-date instead of recreating them
+- allow to build rulesets using rule comment field
+
 ## [5.4.0] - 2024-07-09
 - added `rule_source_id` and `excluded_rules` parameters to `POST /rulestets`.
 - added auto version resolving to all the `/rulesets` endpoints. Version parameters is optional

Makefile

@@ -13,14 +13,15 @@ AWS_REGION = $(shell aws configure get region)
 
 EXECUTOR_IMAGE_NAME := rule-engine-executor  # just dev image name
 EXECUTOR_IMAGE_TAG := latest
-SERVER_IMAGE_NAME := rule-engine
-SERVER_IMAGE_TAG := latest
+SERVER_IMAGE_NAME := public.ecr.aws/x4s4z8e1/syndicate/rule-engine
+SERVER_IMAGE_TAG ?= $(shell PYTHONPATH=./src python -B -c "from src.helpers.__version__ import __version__; print(__version__)")
 
 
 SYNDICATE_EXECUTABLE_PATH ?= $(shell which syndicate)
 SYNDICATE_CONFIG_PATH ?= .syndicate-config-main
 SYNDICATE_BUNDLE_NAME := custodian-service
 
+HELM_REPO_NAME := syndicate
 
 check-syndicate:
 	@if [[ -z "$(SYNDICATE_EXECUTABLE_PATH)" ]]; then echo "No syndicate executable found"; exit 1; fi
@@ -80,17 +81,6 @@ fork-executor-image:
 	# $(DOCKER_EXECUTABLE) build -t $(EXECUTOR_IMAGE_NAME):$(EXECUTOR_IMAGE_TAG) -f src/executor/Dockerfile --build-arg CUSTODIAN_SERVICE_PATH=custodian-as-a-service --build-arg CLOUD_CUSTODIAN_PATH=custodian-custom-core ..
 
 
-open-source-server-image:
-	$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile-opensource .
-
-fork-server-image:
-	$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile .
-
-
-open-source-server-image-to-minikube:
-	eval $(minikube -p minikube docker-env) && \
-	$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile-opensource .
-
 cli-dist:
 	python -m pip install --upgrade build
 	python -m build --sdist cli/
@@ -110,12 +100,6 @@ aws-ecr-push-executor:
 	$(DOCKER_EXECUTABLE) push $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(EXECUTOR_IMAGE_NAME):$(EXECUTOR_IMAGE_TAG)
 
 
-aws-ecr-push-server:
-	export AWS_REGION=$(AWS_REGION) AWS_ACCOUNT_ID=$(AWS_ACCOUNT_ID); \
-	$(DOCKER_EXECUTABLE) tag $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG); \
-	$(DOCKER_EXECUTABLE) push $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
-
-
 syndicate-update-lambdas: check-syndicate
 	SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) build --errors_allowed --bundle_name $(SYNDICATE_BUNDLE_NAME) -F
 	SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) update --update_only_types lambda --update_only_types lambda_layer --bundle_name $(SYNDICATE_BUNDLE_NAME) --replace_output
@@ -135,3 +119,40 @@ syndicate-update-api-gateway: check-syndicate
 syndicate-update-step-functions: check-syndicate
 	# it does not remove the old api gateway
 	SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) deploy --deploy_only_types step_functions --replace_output --bundle_name $(SYNDICATE_BUNDLE_NAME)
+
+
+# images with fork which is default for now. Use src/onprem/Dockerfile-opensource for c7n from open source
+#make image-arm64
+#make image-amd64
+#make push-arm64
+#make push-amd64
+#make image-manifest
+#make push-manifest
+image-arm64:
+	$(DOCKER_EXECUTABLE) build --platform linux/arm64 -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 -f src/onprem/Dockerfile .
+
+image-amd64:
+	$(DOCKER_EXECUTABLE) build --platform linux/amd64 -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64 -f src/onprem/Dockerfile .
+
+
+image-manifest:
+	-$(DOCKER_EXECUTABLE) manifest rm $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
+	$(DOCKER_EXECUTABLE) manifest create $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64
+	$(DOCKER_EXECUTABLE) manifest annotate $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 --arch arm64
+	$(DOCKER_EXECUTABLE) manifest annotate $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64 --arch amd64
+
+push-arm64:
+	$(DOCKER_EXECUTABLE) push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64
+
+
+push-amd64:
+	$(DOCKER_EXECUTABLE) push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64
+
+push-manifest:
+	$(DOCKER_EXECUTABLE) manifest push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
+
+
+push-helm-chart:
+	helm package --dependency-update deployment/helm/rule-engine
+	helm s3 push rule-engine-$(SERVER_IMAGE_TAG).tgz $(HELM_REPO_NAME)
+	-rm rule-engine-$(SERVER_IMAGE_TAG).tgz

cli/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.5.0] - 2024-09-02
+- change job submit command to resolve credentials for tenant from all available places
+- add `--platform`, `--category`, `--service_section` and `--source` fields to `sre ruleset add`
+
+## [5.4.1] - 2024-08-06
+- fix `inappropriate ioctl for device`
+- add environment variables: `SRE_CLI_RESPONSE_FORMAT`, `SRE_CLI_VERBOSE`, `SRE_CLI_NO_PROMPT`
+
 ## [5.4.0] - 2024-07-09
 - renamed `c7n` entrypoint to `sre`
 - add 1 exit codes for all commands that failed

cli/api_module.json

@@ -1,5 +1,5 @@
 {
     "module_name": "srecli",
     "cli_path": "/srecli/group",
-    "mount_point": "/c7n"
+    "mount_point": "/re"
 }

cli/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 sre = "srecli.group.sre:sre"
 
 [tool.setuptools.dynamic]
-version = {attr = "srecli.version.__version__"}
+version = {attr = "srecli.__version__"}
 
 [tool.setuptools.packages.find]
 where = ["."]

cli/srecli/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '5.5.0'

cli/srecli/group/__init__.py

@@ -1,15 +1,15 @@
+import json
+import operator
+import shutil
+import sys
+import urllib.error
 from abc import ABC, abstractmethod
 from datetime import timezone
 from functools import reduce, wraps
 from http import HTTPStatus
-import operator
 from itertools import islice
-import json
-import os
-import sys
 from pathlib import Path
 from typing import Any, Callable, TypedDict, cast
-import urllib.error
 
 import click
 from dateutil.parser import isoparse
@@ -31,9 +31,13 @@
     NEXT_TOKEN_ATTR,
     NO_CONTENT_RESPONSE_MESSAGE,
     NO_ITEMS_TO_DISPLAY_RESPONSE_MESSAGE,
-    JobType
+    JobType,
+    Env,
+    MODULAR_ADMIN,
+    STATUS_ATTR, SUCCESS_STATUS, ERROR_STATUS, CODE_ATTR, TABLE_TITLE_ATTR,
+    REVERT_TO_JSON_MESSAGE, COLUMN_OVERFLOW
 )
-from srecli.service.logger import get_logger, get_user_logger, write_verbose_logs
+from srecli.service.logger import get_logger, enable_verbose_logs
 
 CredentialsProvider = None
 try:
@@ -43,22 +47,7 @@
     pass
 
 
-# modular cli
-MODULAR_ADMIN = 'modules'
-SUCCESS_STATUS = 'SUCCESS'
-ERROR_STATUS = 'FAILED'
-STATUS_ATTR = 'status'
-CODE_ATTR = 'code'
-TABLE_TITLE_ATTR = 'table_title'
-# -----------
-
 _LOG = get_logger(__name__)
-USER_LOG = get_user_logger(__name__)
-
-REVERT_TO_JSON_MESSAGE = 'The command`s response is pretty huge and the ' \
-                         'result table structure can be broken.\nDo you want ' \
-                         'to show the response in the JSON format?'
-COLUMN_OVERFLOW = 'Column has overflown, within the table representation.'
 
 
 class TableException(Exception):
@@ -170,10 +159,12 @@ def wrapper(*args, **kwargs):
             if Path(__file__).parents[3].name == MODULAR_ADMIN:  # TODO check some other way
                 modular_mode = True
 
-            json_view = kwargs.pop('json')
-            verbose = kwargs.pop('verbose')
+            json_view = Env.RESPONSE_FORMAT.get() == 'json' or kwargs.get('json')
+            verbose = Env.VERBOSE.get() or kwargs.get('verbose')  # todo verbose can be enabled earlier if from env
+            kwargs.pop('json', None)
+            kwargs.pop('verbose', None)
             if verbose:
-                write_verbose_logs()
+                enable_verbose_logs()
             ctx = cast(click.Context, click.get_current_context())
             self.update_context(ctx)
             try:
@@ -204,7 +195,10 @@ def wrapper(*args, **kwargs):
                         items_per_column=ctx.obj['config'].items_per_column,
                         attributes_order=self._attributes_order
                     )
-                    table = printer.print(prepared)
+                    table = printer.print(
+                        prepared,
+                        raise_on_overflow=not Env.NO_PROMPT.get()
+                    )
                 except ColumnOverflow as ce:
 
                     _LOG.info(f'Awaiting user to respond to - {ce!r}.')
@@ -253,6 +247,8 @@ def format(self, resp: CustodianResponse) -> dict:
         if resp.code == HTTPStatus.NO_CONTENT:
             return {MESSAGE_ATTR: NO_CONTENT_RESPONSE_MESSAGE}
         elif isinstance(resp.exc, json.JSONDecodeError):
+            if not resp.data and resp.code:
+                return {MESSAGE_ATTR: resp.code.phrase}
             return {MESSAGE_ATTR: f'Invalid JSON received: {resp.exc.msg}'}
         elif isinstance(resp.exc, urllib.error.URLError):
             return {MESSAGE_ATTR: f'Cannot send a request: {resp.exc.reason}'}
@@ -308,7 +304,7 @@ def _format_er(e):
 
     def format(self, resp: CustodianResponse) -> dict:
         base = {
-            CODE_ATTR: resp.code,
+            CODE_ATTR: resp.code or HTTPStatus.SERVICE_UNAVAILABLE.value,
             STATUS_ATTR: SUCCESS_STATUS if resp.ok else ERROR_STATUS,
             TABLE_TITLE_ATTR: self.modular_table_title
         }
@@ -399,7 +395,7 @@ def key(tpl):
         else:
             formatted = self._items_table(data)
 
-        overflow = formatted.index('\n') > os.get_terminal_size().columns
+        overflow = formatted.index('\n') > shutil.get_terminal_size().columns
         if overflow and raise_on_overflow:
             raise ColumnOverflow(table=formatted)
         return formatted
@@ -465,14 +461,14 @@ def response(*args, **kwargs):
 
 # callbacks
 def convert_in_upper_case_if_present(ctx, param, value):
-    if isinstance(value, list | tuple):
+    if isinstance(value, (list, tuple)):
         return [each.upper() for each in value]
     elif value:
         return value.upper()
 
 
 def convert_in_lower_case_if_present(ctx, param, value):
-    if isinstance(value, list):
+    if isinstance(value, (list, tuple)):
         return [each.lower() for each in value]
     elif value:
         return value.lower()
@@ -499,17 +495,6 @@ def build_account_option(**kwargs) -> Callable:
     return click.option('--account_number', '-acc', **params)
 
 
-def build_tenant_display_name_option(**kwargs) -> Callable:
-    params = dict(
-        type=str,
-        required=True,
-        help='The name of the target tenant group',
-        callback=convert_in_lower_case_if_present
-    )
-    params.update(kwargs)
-    return click.option('--tenant_display_name', '-tdn', **params)
-
-
 def build_iso_date_option(*args, **kwargs) -> Callable:
     help_iso = 'ISO 8601 format. Example: 2021-09-22T00:00:00.000000'
     params = dict(type=isoparse, required=False)
@@ -563,7 +548,6 @@ def build_limit_option(**kwargs) -> Callable:
 
 
 tenant_option = build_tenant_option()
-tenant_display_name_option = build_tenant_display_name_option()
 account_option = build_account_option()
 
 optional_job_type_option = build_job_type_option()

cli/srecli/group/integrations.py

@@ -1,7 +1,7 @@
 import click
 
 from srecli.group.integrations_dojo import dojo
-from srecli.group.integrations_sre import sre
+from srecli.group.integrations_re import re
 from srecli.group.integrations_chronicle import chronicle
 
 
@@ -11,5 +11,5 @@ def integrations():
 
 
 integrations.add_command(dojo)
-integrations.add_command(sre)
+integrations.add_command(re)
 integrations.add_command(chronicle)

cli/srecli/group/integrations_sre.py → cli/srecli/group/integrations_re.py

@@ -9,15 +9,15 @@
 from srecli.service.constants import AWS, AZURE, GOOGLE, KUBERNETES
 
 
-@click.group(name='sre')
-def sre():
+@click.group(name='re')
+def re():
     """
     Manages Rule engine integration (self integration for Maestro)
     :return:
     """
 
 
-@sre.command(cls=ViewCommand, name='add')
+@re.command(cls=ViewCommand, name='add')
 @build_tenant_option(multiple=True)
 @click.option('--all_tenants', is_flag=True,
               help='Whether to activate integration for all tenants')
@@ -84,7 +84,7 @@ def add(ctx: ContextObj, tenant_name: tuple[str, ...], all_tenants: bool,
     )
 
 
-@sre.command(cls=ViewCommand, name='describe')
+@re.command(cls=ViewCommand, name='describe')
 @cli_response()
 def describe(ctx: ContextObj, customer_id):
     """
@@ -93,7 +93,7 @@ def describe(ctx: ContextObj, customer_id):
     return ctx['api_client'].sre_describe(customer_id=customer_id)
 
 
-@sre.command(cls=ViewCommand, name='delete')
+@re.command(cls=ViewCommand, name='delete')
 @cli_response()
 def delete(ctx: ContextObj, customer_id):
     """
@@ -102,7 +102,7 @@ def delete(ctx: ContextObj, customer_id):
     return ctx['api_client'].sre_delete(customer_id=customer_id)
 
 
-@sre.command(cls=ViewCommand, name='update')
+@re.command(cls=ViewCommand, name='update')
 @click.option('--add_tenant', '-at', type=str, multiple=True,
               help='Tenants to activate')
 @click.option('--exclude_tenant', '-et', type=str, multiple=True,