Private messaging is in using specified tags and sending only the nes…

.azure-pipelines.yml

@@ -44,9 +44,8 @@ jobs:
       architecture: 'x64'
   # bin directory is in .dockerignore
   - script: |
-      python -m pip install --upgrade pip
-      pip install docker click pytest pyyaml
+      python3 -m pip install --upgrade pip
+      pip3 install docker click pytest pyyaml six
   # build a docker image and sanity test
   - script: |
-      python tools/dev/dockerpkg.py build -t build --verbose --test -i cli
-
+      python3 tools/dev/dockerpkg.py build -t build --verbose --test -i cli

.github/workflows/ci-master.yml

@@ -88,6 +88,11 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -Z \
           -v -f coverage.xml
 
+      - name: License Check
+        if: contains(matrix.tox-target, 'py39')
+        run: |
+          ./.tox/${{ matrix.tox-target }}/bin/python tools/dev/license-check.py
+
   Test-Windows:
     # windows can't use the fast cache technique we use in our matrix builds
     # where we cache the entire tox virtualenv directory, without error.
@@ -116,6 +121,23 @@ jobs:
         run: |
           tox -e py37
 
+  Analyzer:
+    runs-on: ubuntu-latest
+    needs: Lint
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+      - name: Run Bandit
+        run: |
+          python -m pip install bandit
+          make analyzer-bandit
+      - name: Run Semgrep
+        run: |
+          python -m pip install semgrep
+          make analyzer-semgrep
+
   Docs:
     # todo, figure out how to fast cache the tox directory here.
     runs-on: ubuntu-latest

.github/workflows/docker.yml

@@ -22,7 +22,7 @@ jobs:
         # bin directory is in .dockerignore
         run: |
           python -m pip install --upgrade pip
-          pip install docker click pytest pyyaml
+          pip install docker click pytest pyyaml six
           mkdir -p bin
           wget -q -O bin/trivy.tgz https://github.com/aquasecurity/trivy/releases/download/v0.5.4/trivy_0.5.4_Linux-64bit.tar.gz
           cd bin && tar xzf trivy.tgz

.vscode/tasks.json

@@ -1,7 +1,7 @@
 {
     // See https://go.microsoft.com/fwlink/?LinkId=733558
     // for the documentation about the tasks.json format
-    "version": "0.1.0",
+    "version": "2.0.0",
     "command": "make",
     "isShellCommand": true,
     "echoCommand": true,
@@ -13,4 +13,4 @@
         {"taskName": "coverage",
          "args": ["coverage"]}
     ]
-}
+}

Curtis_upload_test

@@ -0,0 +1 @@
+make sure this works

Makefile

@@ -107,3 +107,28 @@ lint:
 
 clean:
 	rm -rf .tox .Python bin include lib pip-selfcheck.json
+
+analyzer-bandit:
+	bandit -i -s B101,B311 \
+	-r tools/c7n_azure/c7n_azure \
+	 tools/c7n_gcp/c7n_gcp \
+	 tools/c7n_terraform/c7n_terraform \
+	 tools/c7n_guardian/c7n_guardian \
+	 tools/c7n_org/c7n_org \
+	 tools/c7n_mailer/c7n_mailer \
+	 tools/c7n_policystream/policystream.py \
+	 tools/c7n_trailcreator/c7n_trailcreator \
+	 c7n
+
+
+analyzer-semgrep:
+	semgrep --error --verbose --config p/security-audit \
+	 tools/c7n_azure/c7n_azure \
+	 tools/c7n_gcp/c7n_gcp \
+	 tools/c7n_terraform/c7n_terraform \
+	 tools/c7n_guardian/c7n_guardian \
+	 tools/c7n_org/c7n_org \
+	 tools/c7n_mailer/c7n_mailer \
+	 tools/c7n_policystream/policystream.py \
+	 tools/c7n_trailcreator/c7n_trailcreator \
+	 c7n

c7n/cache.py

@@ -3,7 +3,7 @@
 """Provide basic caching services to avoid extraneous queries over
 multiple policies on the same resource type.
 """
-import pickle
+import pickle  # nosec nosemgrep
 
 import os
 import logging
@@ -65,10 +65,10 @@ def load(self):
         return True
 
     def get(self, key):
-        return self.data.get(pickle.dumps(key))
+        return self.data.get(pickle.dumps(key))  # nosemgrep
 
     def save(self, key, data):
-        self.data[pickle.dumps(key)] = data
+        self.data[pickle.dumps(key)] = data  # nosemgrep
 
     def size(self):
         return sum(map(len, self.data.values()))
@@ -86,7 +86,7 @@ def __init__(self, config):
         self.data = {}
 
     def get(self, key):
-        k = pickle.dumps(key)
+        k = pickle.dumps(key)  # nosemgrep
         return self.data.get(k)
 
     def load(self):
@@ -98,17 +98,17 @@ def load(self):
                 return False
             with open(self.cache_path, 'rb') as fh:
                 try:
-                    self.data = pickle.load(fh)
+                    self.data = pickle.load(fh)  # nosec nosemgrep
                 except EOFError:
                     return False
             log.debug("Using cache file %s" % self.cache_path)
             return True
 
     def save(self, key, data):
         try:
-            with open(self.cache_path, 'wb') as fh:
-                self.data[pickle.dumps(key)] = data
-                pickle.dump(self.data, fh, protocol=2)
+            with open(self.cache_path, 'wb') as fh:  # nosec
+                self.data[pickle.dumps(key)] = data  # nosemgrep
+                pickle.dump(self.data, fh, protocol=2)  # nosemgrep
         except Exception as e:
             log.warning("Could not save cache %s err: %s" % (
                 self.cache_path, e))

c7n/commands.py

@@ -205,7 +205,8 @@ def validate(options):
 
         with open(config_file) as fh:
             if fmt in ('yml', 'yaml', 'json'):
-                data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader)
+                # our loader is safe loader derived.
+                data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader)  # nosec nosemgrep
             else:
                 log.error("The config file must end in .json, .yml or .yaml.")
                 raise ValueError("The config file must end in .json, .yml or .yaml.")

c7n/filters/core.py

@@ -581,7 +581,7 @@ def match(self, i):
                 return op(r, v)
             except TypeError:
                 return False
-        elif r == self.v:
+        elif r == v:
             return True
 
         return False

c7n/filters/kms.py

@@ -1,9 +1,8 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
-from botocore.exceptions import ClientError
 from .core import ValueFilter
 from .related import RelatedResourceFilter
-from c7n.utils import local_session, type_schema
+from c7n.utils import type_schema
 
 
 class KmsRelatedFilter(RelatedResourceFilter):
@@ -13,15 +12,31 @@ class KmsRelatedFilter(RelatedResourceFilter):
 
     :example:
 
+    Match a specific key alias:
+
         .. code-block:: yaml
 
             policies:
                 - name: dms-encrypt-key-check
                   resource: dms-instance
                   filters:
                     - type: kms-key
-                      key: c7n:AliasName
+                      key: "c7n:AliasName"
                       value: alias/aws/dms
+
+    Or match against native key attributes such as ``KeyManager``, which
+    more explicitly distinguishes between ``AWS`` and ``CUSTOMER``-managed
+    keys. The above policy can also be written as:
+
+        .. code-block:: yaml
+
+            policies:
+                - name: dms-aws-managed-key
+                  resource: dms-instance
+                  filters:
+                    - type: kms-key
+                      key: KeyManager
+                      value: AWS
     """
 
     schema = type_schema(
@@ -31,6 +46,24 @@ class KmsRelatedFilter(RelatedResourceFilter):
     RelatedResource = "c7n.resources.kms.Key"
     AnnotationKey = "matched-kms-key"
 
+    def get_related(self, resources):
+        resource_manager = self.get_resource_manager()
+        related_ids = self.get_related_ids(resources)
+        if len(related_ids) < self.FetchThreshold:
+            related = resource_manager.get_resources(list(related_ids))
+        else:
+            related = resource_manager.resources()
+        related_map = {}
+
+        # A resource's key property may point to an explicit ID or a key alias.
+        # Be sure that a related key lookup covers both cases.
+        for r in related:
+            related_map[r['KeyId']] = r
+            for alias in r.get('AliasNames', []):
+                related_map[alias] = r
+
+        return related_map
+
     def get_related_ids(self, resources):
         related_ids = super().get_related_ids(resources)
         normalized_ids = []
@@ -42,13 +75,10 @@ def get_related_ids(self, resources):
         return normalized_ids
 
     def process(self, resources, event=None):
-        client = local_session(self.manager.session_factory).client('kms')
         related = self.get_related(resources)
         for r in related.values():
-            try:
-                alias_info = self.manager.retry(client.list_aliases, KeyId=r.get('KeyId'))
-            except ClientError as e:
-                self.log.warning(e)
-                continue
-            r['c7n:AliasName'] = alias_info.get('Aliases')[0].get('AliasName', '')
+            # `AliasNames` is set when we fetch keys, but only for keys
+            # which have aliases defined. Fall back to an empty string
+            # to avoid lookup errors in filters.
+            r['c7n:AliasName'] = r.get('AliasNames', ('',))[0]
         return [r for r in resources if self.process_resource(r, related)]

c7n/filters/offhours.py

@@ -29,6 +29,9 @@
  - **weekends-only**: default false, whether to turn the resource off only on
    the weekend
  - **default_tz**: which timezone to utilize when evaluating time **(REQUIRED)**
+ - **fallback-schedule**: If a resource doesn't support tagging or doesn't provide
+   a tag you can supply a default schedule that will be used. When the tag is provided
+   this will be ignored. See :ref:`ScheduleParser Time Specifications <scheduleparser-time-spec>`.
  - **tag**: which resource tag name to use for per-resource configuration
    (schedule and timezone overrides and opt-in/opt-out); default is
    ``maid_offhours``.
@@ -61,6 +64,7 @@
            onhour: 8
            offhour: 20
 
+
 Tag Based Configuration
 =======================
 
@@ -94,6 +98,9 @@
     supported by :py:class:`c7n.filters.offhours.ScheduleParser` as described
     in the next section.
 
+
+.. _scheduleparser-time-spec:
+
 ScheduleParser Time Specifications
 ----------------------------------
 
@@ -255,6 +262,7 @@ class Time(Filter):
         'properties': {
             'tag': {'type': 'string'},
             'default_tz': {'type': 'string'},
+            'fallback_schedule': {'type': 'string'},
             'weekends': {'type': 'boolean'},
             'weekends-only': {'type': 'boolean'},
             'opt-out': {'type': 'boolean'},
@@ -315,6 +323,7 @@ def __init__(self, data, manager=None):
         self.weekends_only = self.data.get('weekends-only', False)
         self.opt_out = self.data.get('opt-out', False)
         self.tag_key = self.data.get('tag', self.DEFAULT_TAG).lower()
+        self.fallback_schedule = self.data.get('fallback-schedule', None)
         self.default_schedule = self.get_default_schedule()
         self.parser = ScheduleParser(self.default_schedule)
 
@@ -438,12 +447,12 @@ def match(self, now, schedule):
     def get_tag_value(self, i):
         """Get the resource's tag value specifying its schedule."""
         # Look for the tag, Normalize tag key and tag value
-        found = False
+        found = self.fallback_schedule
         for t in i.get('Tags', ()):
             if t['Key'].lower() == self.tag_key:
                 found = t['Value']
                 break
-        if found is False:
+        if found in (False, None):
             return False
         # enforce utf8, or do translate tables via unicode ord mapping
         value = found.lower().encode('utf8').decode('utf8')

c7n/handler.py

@@ -100,7 +100,7 @@ def init_config(policy_config):
 
     # a cli local directory doesn't translate to lambda
     if not exec_options.get('output_dir', '').startswith('s3'):
-        exec_options['output_dir'] = '/tmp'
+        exec_options['output_dir'] = '/tmp'  # nosec
 
     account_id = None
     # we can source account id from the cli parameters to avoid the sts call

c7n/mu.py

@@ -1078,6 +1078,8 @@ def render_event_pattern(self):
                 })
             if self.data.get('categories', []):
                 payload['detail']['eventTypeCategory'] = self.data['categories']
+            if not payload['detail']:
+                payload.pop('detail')
         elif event_type == 'hub-finding':
             payload['source'] = ['aws.securityhub']
             payload['detail-type'] = ['Security Hub Findings - Imported']
@@ -1160,13 +1162,13 @@ def update(self, func):
     def pause(self, func):
         try:
             self.client.disable_rule(Name=func.name)
-        except Exception:
+        except ClientError:
             pass
 
     def resume(self, func):
         try:
             self.client.enable_rule(Name=func.name)
-        except Exception:
+        except ClientError:
             pass
 
     def remove(self, func):
@@ -1663,7 +1665,7 @@ def delta(rule, params):
         if ('MaximumExecutionFrequency' in params and
                 rule['MaximumExecutionFrequency'] != params['MaximumExecutionFrequency']):
             return True
-        if rule.get('Description', '') != rule.get('Description', ''):
+        if rule.get('Description', '') != params.get('Description', ''):
             return True
         return False
 

c7n/output.py

@@ -330,9 +330,9 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None):
-        self.leave_log()
         if exc_type is not None:
             log.exception("Error while executing policy")
+        self.leave_log()
 
     def join_log(self):
         self.handler = self.get_handler()

c7n/query.py

@@ -364,7 +364,11 @@ def _load_resource_tags(self, resource, item):
             if isinstance(stags, str):
                 stags = json.loads(stags)
             if isinstance(stags, list):
-                resource['Tags'] = [{u'Key': t['key'], u'Value': t['value']} for t in stags]
+                resource['Tags'] = [
+                    {u'Key': t.get('key', t.get('tagKey')),
+                     u'Value': t.get('value', t.get('tagValue'))}
+                    for t in stags
+                ]
             elif isinstance(stags, dict):
                 resource['Tags'] = [{u'Key': k, u'Value': v} for k, v in stags.items()]