diff --git a/config/settings.yml b/config/settings.yml
index 0d71b794..1e1fa7ac 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -18,6 +18,9 @@ bugzilla:
   product:
 grafana:
   url:
+labels:
+  unassignable: {}
+  unremovable: []
 
 # Worker settings
 diff_content_checker:
diff --git a/config/settings/test.yml b/config/settings/test.yml
index 9e314711..e4da3f6e 100644
--- a/config/settings/test.yml
+++ b/config/settings/test.yml
@@ -1,3 +1,9 @@
+:labels:
+  :unassignable:
+    jansa/yes: jansa/yes?
+  :unremovable:
+  - jansa/no
+  - jansa/yes
 # In test, turn everything on by default
 #   NOTE: Using empty string is a HACK until we can get
 #   https://github.com/danielsdeleo/deep_merge/pull/33 released via the
diff --git a/lib/github_service/commands/add_label.rb b/lib/github_service/commands/add_label.rb
index 694038ce..3baaa4c2 100644
--- a/lib/github_service/commands/add_label.rb
+++ b/lib/github_service/commands/add_label.rb
@@ -3,9 +3,9 @@ module Commands
     class AddLabel < Base
       include IsTeamMember
 
-      UNASSIGNABLE = {
-        "jansa/yes" => "jansa/yes?"
-      }.freeze
+      def unassignable_labels
+        @unassignable_labels ||= Settings.labels.unassignable.to_h.stringify_keys
+      end
 
       private
 
@@ -57,7 +57,7 @@ def correct_invalid_labels(valid_labels, invalid_labels)
 
       def handle_unassignable_labels(valid_labels)
         valid_labels.map! do |label|
-          UNASSIGNABLE.key?(label) ? UNASSIGNABLE[label] : label
+          unassignable_labels.key?(label) ? unassignable_labels[label] : label
         end
       end
 
diff --git a/lib/github_service/commands/remove_label.rb b/lib/github_service/commands/remove_label.rb
index 3cc57871..180391d6 100644
--- a/lib/github_service/commands/remove_label.rb
+++ b/lib/github_service/commands/remove_label.rb
@@ -3,8 +3,6 @@ module Commands
     class RemoveLabel < Base
       include IsTeamMember
 
-      UNREMOVABLE = %w[jansa/yes jansa/no].freeze
-
       alias_as 'rm_label'
 
       private
@@ -40,7 +38,7 @@ def extract_label_names(value)
 
       def process_extracted_labels(issuer, valid_labels, _invalid_labels, unremovable)
         unless triage_member?(issuer)
-          valid_labels.each { |label| unremovable << label if UNREMOVABLE.include?(label) }
+          valid_labels.each { |label| unremovable << label if Settings.labels.unremovable.include?(label) }
           unremovable.each  { |label| valid_labels.delete(label) }
         end
       end