diff --git a/elliottlib/bzutil.py b/elliottlib/bzutil.py
index 273222c8..2d0eef92 100644
--- a/elliottlib/bzutil.py
+++ b/elliottlib/bzutil.py
@@ -227,6 +227,13 @@ def component(self):
     def status(self):
         return self.bug.fields.status.name
 
+    @property
+    def security_level(self):
+        try:
+            return self.bug.fields.security
+        except AttributeError:
+            return None
+
     def is_tracker_bug(self):
         has_keywords = set(constants.TRACKER_BUG_KEYWORDS).issubset(set(self.keywords))
         has_whiteboard_component = bool(self.whiteboard_component)
diff --git a/elliottlib/cli/attach_bugs_cli.py b/elliottlib/cli/attach_bugs_cli.py
index e535550f..0c954284 100644
--- a/elliottlib/cli/attach_bugs_cli.py
+++ b/elliottlib/cli/attach_bugs_cli.py
@@ -61,9 +61,9 @@ def attach_bugs_cli(runtime: Runtime, advisory, default_advisory_type, bug_ids,
 
     jira_ids, bz_ids = get_jira_bz_bug_ids(bug_ids)
     if jira_ids:
-        attach_bugs(runtime, advisory, jira_ids, report, output, noop, runtime.bug_trackers('jira'))
+        attach_bugs(runtime, advisory, jira_ids, report, output, noop, runtime.get_bug_tracker('jira'))
     if bz_ids:
-        attach_bugs(runtime, advisory, bz_ids, report, output, noop, runtime.bug_trackers('bugzilla'))
+        attach_bugs(runtime, advisory, bz_ids, report, output, noop, runtime.get_bug_tracker('bugzilla'))
 
 
 def attach_bugs(runtime, advisory, bug_ids, report, output, noop, bug_tracker):
diff --git a/elliottlib/cli/attach_cve_flaws_cli.py b/elliottlib/cli/attach_cve_flaws_cli.py
index 3feaae8f..12c2b5be 100644
--- a/elliottlib/cli/attach_cve_flaws_cli.py
+++ b/elliottlib/cli/attach_cve_flaws_cli.py
@@ -59,7 +59,7 @@ async def attach_cve_flaws_cli(runtime: Runtime, advisory_id: int, noop: bool, d
     else:
         advisories = [advisory_id]
     exit_code = 0
-    flaw_bug_tracker = runtime.bug_trackers('bugzilla')
+    flaw_bug_tracker = runtime.get_bug_tracker('bugzilla')
     errata_config = runtime.get_errata_config()
     errata_api = AsyncErrataAPI(errata_config.get("server", constants.errata_url))
     brew_api = runtime.build_retrying_koji_client()
@@ -68,7 +68,7 @@ async def attach_cve_flaws_cli(runtime: Runtime, advisory_id: int, noop: bool, d
         advisory = Erratum(errata_id=advisory_id)
 
         attached_trackers = []
-        for bug_tracker in [runtime.bug_trackers('jira'), runtime.bug_trackers('bugzilla')]:
+        for bug_tracker in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
             attached_trackers.extend(get_attached_trackers(advisory, bug_tracker, runtime.logger))
 
         tracker_flaws, flaw_bugs = get_flaws(flaw_bug_tracker, attached_trackers, brew_api, runtime.logger)
diff --git a/elliottlib/cli/create_placeholder_cli.py b/elliottlib/cli/create_placeholder_cli.py
index 49aab5ae..405a5f03 100644
--- a/elliottlib/cli/create_placeholder_cli.py
+++ b/elliottlib/cli/create_placeholder_cli.py
@@ -45,7 +45,7 @@ def create_placeholder_cli(runtime, kind, advisory_id, default_advisory_type, no
     if not kind:
         raise click.BadParameter("--kind must be specified when not using --use-default-advisory")
 
-    create_placeholder(kind, advisory_id, runtime.bug_trackers('jira'), noop)
+    create_placeholder(kind, advisory_id, runtime.get_bug_tracker('jira'), noop)
 
 
 def create_placeholder(kind, advisory_id, bug_tracker, noop):
diff --git a/elliottlib/cli/create_textonly_cli.py b/elliottlib/cli/create_textonly_cli.py
index 252d422d..084c286b 100644
--- a/elliottlib/cli/create_textonly_cli.py
+++ b/elliottlib/cli/create_textonly_cli.py
@@ -69,11 +69,11 @@ def create_textonly_cli(runtime, errata_type, date, assigned_to, manager, packag
     # we give priority to jira in case both are in use
     if runtime.use_jira:
         create_textonly(runtime, errata_type, date, assigned_to, manager, package_owner, topic, synopsis,
-                        description, solution, bugtitle, bugdescription, yes, runtime.bug_trackers('jira'))
+                        description, solution, bugtitle, bugdescription, yes, runtime.get_bug_tracker('jira'))
 
     else:
         create_textonly(runtime, errata_type, date, assigned_to, manager, package_owner, topic, synopsis,
-                        description, solution, bugtitle, bugdescription, yes, runtime.bug_trackers('bugzilla'))
+                        description, solution, bugtitle, bugdescription, yes, runtime.get_bug_tracker('bugzilla'))
 
 
 def create_textonly(runtime, errata_type, date, assigned_to, manager, package_owner, topic, synopsis, description,
diff --git a/elliottlib/cli/find_bugs_blocker_cli.py b/elliottlib/cli/find_bugs_blocker_cli.py
index da351aa8..55e936d0 100644
--- a/elliottlib/cli/find_bugs_blocker_cli.py
+++ b/elliottlib/cli/find_bugs_blocker_cli.py
@@ -61,7 +61,7 @@ def find_bugs_blocker_cli(runtime: Runtime, include_status, exclude_status, outp
     find_bugs_obj.include_status(include_status)
     find_bugs_obj.exclude_status(exclude_status)
     exit_code = 0
-    for b in [runtime.bug_trackers('jira'), runtime.bug_trackers('bugzilla')]:
+    for b in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
         try:
             find_bugs_blocker(runtime, output, find_bugs_obj, b)
         except Exception as e:
diff --git a/elliottlib/cli/find_bugs_kernel_cli.py b/elliottlib/cli/find_bugs_kernel_cli.py
index ed15bea8..a5896fe4 100644
--- a/elliottlib/cli/find_bugs_kernel_cli.py
+++ b/elliottlib/cli/find_bugs_kernel_cli.py
@@ -48,9 +48,9 @@ async def run(self):
             logger.warning("kernel_bug_sweep is not defined in bug.yml")
             return
         config = KernelBugSweepConfig.parse_obj(raw_config)
-        jira_tracker = self._runtime.bug_trackers("jira")
+        jira_tracker = self._runtime.get_bug_tracker("jira")
         jira_client: JIRA = jira_tracker._client
-        bz_tracker = self._runtime.bug_trackers("bugzilla")
+        bz_tracker = self._runtime.get_bug_tracker("bugzilla")
         bz_client: Bugzilla = bz_tracker._client
         koji_api = self._runtime.build_retrying_koji_client(caching=True)
 
diff --git a/elliottlib/cli/find_bugs_kernel_clones_cli.py b/elliottlib/cli/find_bugs_kernel_clones_cli.py
index 7ba5e9f5..6aa28a3e 100644
--- a/elliottlib/cli/find_bugs_kernel_clones_cli.py
+++ b/elliottlib/cli/find_bugs_kernel_clones_cli.py
@@ -39,7 +39,7 @@ async def run(self):
             logger.warning("kernel_bug_sweep is not defined in bug.yml")
             return
         config = KernelBugSweepConfig.parse_obj(raw_config)
-        jira_tracker = self._runtime.bug_trackers("jira")
+        jira_tracker = self._runtime.get_bug_tracker("jira")
         jira_client: JIRA = jira_tracker._client
         koji_api = self._runtime.build_retrying_koji_client(caching=True)
 
diff --git a/elliottlib/cli/find_bugs_qe_cli.py b/elliottlib/cli/find_bugs_qe_cli.py
index b85f0195..50fed816 100644
--- a/elliottlib/cli/find_bugs_qe_cli.py
+++ b/elliottlib/cli/find_bugs_qe_cli.py
@@ -35,7 +35,7 @@ def find_bugs_qe_cli(runtime: Runtime, noop):
     runtime.initialize()
     find_bugs_obj = FindBugsQE()
     exit_code = 0
-    for b in [runtime.bug_trackers('jira'), runtime.bug_trackers('bugzilla')]:
+    for b in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
         try:
             find_bugs_qe(runtime, find_bugs_obj, noop, b)
         except Exception as e:
@@ -59,11 +59,21 @@ def find_bugs_qe(runtime, find_bugs_obj, noop, bug_tracker):
         f" expected in the next created {major_version}.{minor_version} nightly and release.")
     for bug in bugs:
         updated = bug_tracker.update_bug_status(bug, 'ON_QA', comment=release_comment, noop=noop)
-        if updated and bug.is_tracker_bug():
-            # leave a special comment for QE
-            comment = """Note for QE:
-This is a CVE bug. Please plan on verifying this bug ASAP.
-A CVE bug shouldn't be dropped from an advisory if QE doesn't have enough time to verify.
-Contact ProdSec if you have questions.
-"""
-            bug_tracker.add_comment(bug.id, comment, private=True, noop=noop)
+        if updated:
+            if bug.is_tracker_bug():
+                # leave a special comment for QE
+                comment = """Note for QE:
+    This is a CVE bug. Please plan on verifying this bug ASAP.
+    A CVE bug shouldn't be dropped from an advisory if QE doesn't have enough time to verify.
+    Contact ProdSec if you have questions.
+    """
+                bug_tracker.add_comment(bug.id, comment, private=True, noop=noop)
+
+            elif bug_tracker.type == 'jira':
+                # If a security level is specified, the bug won't be visible on advisories
+                # Make this explicit in the bug comment. Not applicable for security trackers/flaw bugs
+                security_level = bug.security_level
+                if security_level:
+                    comment = "This is not a public issue, the customer visible advisory will not link the fix." \
+                              "Setting the Security Level to public before the advisory ships will have it included"
+                    bug_tracker.add_comment(bug.id, comment, private=True, noop=noop)
diff --git a/elliottlib/cli/find_bugs_sweep_cli.py b/elliottlib/cli/find_bugs_sweep_cli.py
index 5fed5d6c..465a6866 100644
--- a/elliottlib/cli/find_bugs_sweep_cli.py
+++ b/elliottlib/cli/find_bugs_sweep_cli.py
@@ -127,7 +127,7 @@ async def find_bugs_sweep_cli(runtime: Runtime, advisory_id, default_advisory_ty
 
     bugs: type_bug_list = []
     errors = []
-    for b in [runtime.bug_trackers('jira'), runtime.bug_trackers('bugzilla')]:
+    for b in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
         try:
             bugs.extend(await find_and_attach_bugs(runtime, advisory_id, default_advisory_type, major_version, find_bugs_obj,
                         output, brew_event, noop, count_advisory_attach_flags, b))
diff --git a/elliottlib/cli/remove_bugs_cli.py b/elliottlib/cli/remove_bugs_cli.py
index 19f04f0f..ab8005aa 100644
--- a/elliottlib/cli/remove_bugs_cli.py
+++ b/elliottlib/cli/remove_bugs_cli.py
@@ -67,9 +67,9 @@ def remove_bugs_cli(runtime, advisory_id, default_advisory_type, bug_ids, remove
         bz_ids = set(bz_ids) & set(attached_bz_ids)
 
     if jira_ids:
-        remove_bugs(advisory, jira_ids, runtime.bug_trackers('jira'), noop)
+        remove_bugs(advisory, jira_ids, runtime.get_bug_tracker('jira'), noop)
     if bz_ids:
-        remove_bugs(advisory, bz_ids, runtime.bug_trackers('bugzilla'), noop)
+        remove_bugs(advisory, bz_ids, runtime.get_bug_tracker('bugzilla'), noop)
 
 
 def remove_bugs(advisory, bug_ids, bug_tracker, noop):
diff --git a/elliottlib/cli/repair_bugs_cli.py b/elliottlib/cli/repair_bugs_cli.py
index 25867cf8..61d33910 100644
--- a/elliottlib/cli/repair_bugs_cli.py
+++ b/elliottlib/cli/repair_bugs_cli.py
@@ -114,10 +114,10 @@ def repair_bugs_cli(runtime, advisory_id, auto, id, original_state, new_state, c
 
     if jira_ids:
         repair_bugs(jira_ids, original_state, new_state, comment, close_placeholder, noop,
-                    runtime.bug_trackers('jira'))
+                    runtime.get_bug_tracker('jira'))
     if bz_ids:
         repair_bugs(bz_ids, original_state, new_state, comment, close_placeholder, noop,
-                    runtime.bug_trackers('bugzilla'))
+                    runtime.get_bug_tracker('bugzilla'))
 
 
 def repair_bugs(bug_ids, original_state, new_state, comment, close_placeholder, noop, bug_tracker: BugTracker):
diff --git a/elliottlib/cli/verify_attached_bugs_cli.py b/elliottlib/cli/verify_attached_bugs_cli.py
index 365fdd53..6b655ea9 100644
--- a/elliottlib/cli/verify_attached_bugs_cli.py
+++ b/elliottlib/cli/verify_attached_bugs_cli.py
@@ -139,7 +139,7 @@ async def verify_bugs(runtime, verify_bug_status, output, no_verify_blocking_bug
     find_bugs_obj = FindBugsSweep(cve_only=False)
     ocp_bugs = []
     logger.info(f'Using {runtime.assembly} assembly to search bugs')
-    for b in [runtime.bug_trackers('jira'), runtime.bug_trackers('bugzilla')]:
+    for b in [runtime.get_bug_tracker('jira'), runtime.get_bug_tracker('bugzilla')]:
         bugs = find_bugs_obj.search(bug_tracker_obj=b, verbose=runtime.debug)
         logger.info(f"Found {len(bugs)} {b.type} bugs: {[b.id for b in bugs]}")
         ocp_bugs.extend(bugs)
@@ -159,7 +159,7 @@ class BugValidator:
 
     def __init__(self, runtime: Runtime, output: str = 'text'):
         self.runtime = runtime
-        self.target_releases: List[str] = runtime.bug_trackers('jira').config['target_release']
+        self.target_releases: List[str] = runtime.get_bug_tracker('jira').config['target_release']
         self.et_data: Dict[str, Any] = runtime.get_errata_config()
         self.errata_api = AsyncErrataAPI(self.et_data.get("server", constants.errata_url))
         self.problems: List[str] = []
@@ -219,7 +219,7 @@ async def verify_attached_flaws(self, advisory_bugs: Dict[int, List[Bug]]):
         await asyncio.gather(*futures)
 
     async def _verify_attached_flaws_for(self, advisory_id: int, attached_trackers: Iterable[Bug], attached_flaws: Iterable[Bug]):
-        flaw_bug_tracker = self.runtime.bug_trackers('bugzilla')
+        flaw_bug_tracker = self.runtime.get_bug_tracker('bugzilla')
         brew_api = self.runtime.build_retrying_koji_client()
         tracker_flaws, first_fix_flaw_bugs = get_flaws(flaw_bug_tracker, attached_trackers, brew_api,
                                                        self.runtime.logger)
@@ -309,7 +309,7 @@ def get_attached_bugs(self, advisory_ids: List[str]) -> Dict[int, Set[Bug]]:
 
         attached_bug_map = {advisory_id: set() for advisory_id in advisory_ids}
         for bug_tracker_type in ['jira', 'bugzilla']:
-            bug_tracker = self.runtime.bug_trackers(bug_tracker_type)
+            bug_tracker = self.runtime.get_bug_tracker(bug_tracker_type)
             advisory_bug_id_map = {advisory.errata_id: bug_tracker.advisory_bug_ids(advisory)
                                    for advisory in advisories}
             bug_map = bug_tracker.get_bugs_map([bug_id for bug_list in advisory_bug_id_map.values()
@@ -349,16 +349,16 @@ def is_next_target(target_v):
             return pattern.match(target_v) and minor_version_tuple(target_v) == next_version
 
         def managed_by_art(b: Bug):
-            components_not_managed_by_art = self.runtime.bug_trackers('jira').component_filter()
+            components_not_managed_by_art = self.runtime.get_bug_tracker('jira').component_filter()
             return b.component not in components_not_managed_by_art
 
         # retrieve blockers and filter to those with correct product and target version
         blockers = []
         if jira_ids:
-            blockers.extend(self.runtime.bug_trackers('jira')
+            blockers.extend(self.runtime.get_bug_tracker('jira')
                             .get_bugs(jira_ids))
         if bz_ids:
-            blockers.extend(self.runtime.bug_trackers('bugzilla')
+            blockers.extend(self.runtime.get_bug_tracker('bugzilla')
                             .get_bugs(bz_ids))
         logger.debug(f"Candidate Blocker bugs found: {[b.id for b in blockers]}")
         blocking_bugs = {}
diff --git a/elliottlib/runtime.py b/elliottlib/runtime.py
index 8e1d0770..9fd3de78 100644
--- a/elliottlib/runtime.py
+++ b/elliottlib/runtime.py
@@ -18,7 +18,7 @@
 from elliottlib.imagecfg import ImageMetadata
 from elliottlib.model import Missing, Model
 from elliottlib.rpmcfg import RPMMetadata
-from elliottlib.bzutil import BugzillaBugTracker, JIRABugTracker
+from elliottlib.bzutil import BugTracker, BugzillaBugTracker, JIRABugTracker
 
 
 def remove_tmp_working_dir(runtime):
@@ -284,7 +284,7 @@ def image_metas(self):
     def rpm_metas(self):
         return list(self.rpm_map.values())
 
-    def bug_trackers(self, bug_tracker_type):
+    def get_bug_tracker(self, bug_tracker_type) -> BugTracker:
         if bug_tracker_type in self._bug_trackers:
             return self._bug_trackers[bug_tracker_type]
         if bug_tracker_type == 'bugzilla':
diff --git a/tests/test_find_bugs_kernel_cli.py b/tests/test_find_bugs_kernel_cli.py
index 1eb7b376..b242caf9 100644
--- a/tests/test_find_bugs_kernel_cli.py
+++ b/tests/test_find_bugs_kernel_cli.py
@@ -351,6 +351,6 @@ async def test_run_with_specified_trackers(
         cli = FindBugsKernelCli(
             runtime=runtime, trackers=["TRACKER-999"], clone=True, reconcile=True, comment=True, dry_run=False)
         await cli.run()
-        _comment_on_tracker.assert_called_once_with(ANY, jira_client.issue.return_value, ANY, ANY)
+        _comment_on_tracker.assert_called_once()
         _clone_bugs.assert_called_once_with(ANY, _find_bugs.return_value, ANY)
         _find_kmaint_trackers.assert_not_called()
diff --git a/tests/test_find_bugs_kernel_clones_cli.py b/tests/test_find_bugs_kernel_clones_cli.py
index caa983b6..588726bd 100644
--- a/tests/test_find_bugs_kernel_clones_cli.py
+++ b/tests/test_find_bugs_kernel_clones_cli.py
@@ -163,7 +163,6 @@ async def test_run_without_specified_issues(self, _search_for_jira_issues: Mock,
                 },
             }
         )
-        jira_client = runtime.bug_trackers.return_value._client
         found_issues = [
             MagicMock(spec=Issue, **{
                 "key": "FOO-1", "fields": MagicMock(),
@@ -189,7 +188,7 @@ async def test_run_without_specified_issues(self, _search_for_jira_issues: Mock,
         cli = FindBugsKernelClonesCli(
             runtime=runtime, trackers=[], issues=[], move=True, comment=True, dry_run=False)
         await cli.run()
-        _update_jira_issues.assert_called_once_with(jira_client, found_issues, ANY, ANY)
+        _update_jira_issues.assert_called_once()
         expected_report = {
             'jira_issues': [
                 {'key': 'FOO-1', 'summary': 'Fake bug 1', 'status': 'New'},
@@ -226,7 +225,6 @@ async def test_run_with_specified_issues(self, _search_for_jira_issues: Mock, _u
                 },
             }
         )
-        jira_client = runtime.bug_trackers.return_value._client
         found_issues = [
             MagicMock(spec=Issue, **{
                 "key": "FOO-1", "fields": MagicMock(),
@@ -252,7 +250,7 @@ async def test_run_with_specified_issues(self, _search_for_jira_issues: Mock, _u
         cli = FindBugsKernelClonesCli(
             runtime=runtime, trackers=[], issues=["FOO-1", "FOO-2", "FOO-3"], move=True, comment=True, dry_run=False)
         await cli.run()
-        _update_jira_issues.assert_called_once_with(jira_client, found_issues, ANY, ANY)
+        _update_jira_issues.assert_called_once()
         expected_report = {
             'jira_issues': [
                 {'key': 'FOO-1', 'summary': 'Fake bug 1', 'status': 'New'},