diff --git a/.github/workflows/generic-pr.yaml b/.github/workflows/generic-pr.yaml
index b1bcc9b440b06..1c54b3c864136 100644
--- a/.github/workflows/generic-pr.yaml
+++ b/.github/workflows/generic-pr.yaml
@@ -46,18 +46,4 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/workflows/generic-pr/label-config.yml
-          sync-labels: true
-      - name: Label changes scope (Camel 4)
-        if: github.ref == 'refs/heads/main'
-        uses: actions/labeler@v5.0.0
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          configuration-path: .github/workflows/generic-pr/label-config-main.yml
-          sync-labels: true
-      - name: Label changes scope (Camel 3)
-        if: github.ref == 'refs/heads/camel-3.x'
-        uses: actions/labeler@v5.0.0
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          configuration-path: .github/workflows/generic-pr/label-config-camel-3.yml
-          sync-labels: true
+          sync-labels: true
\ No newline at end of file
diff --git a/.github/workflows/generic-pr/label-config-camel-3.yml b/.github/workflows/generic-pr/label-config-camel-3.yml
deleted file mode 100644
index e9de5346af769..0000000000000
--- a/.github/workflows/generic-pr/label-config-camel-3.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-camel-3:
-  - changed-files:
-    - any-glob-to-any-file:
-      - '**/*'
diff --git a/.github/workflows/generic-pr/label-config-main.yml b/.github/workflows/generic-pr/label-config-main.yml
deleted file mode 100644
index 0ca2960f0e0e4..0000000000000
--- a/.github/workflows/generic-pr/label-config-main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-camel-4:
-  - changed-files:
-      - any-glob-to-any-file:
-        - '**/*'
diff --git a/.github/workflows/generic-pr/label-config.yml b/.github/workflows/generic-pr/label-config.yml
index 6c469b2817414..37cd751a81d26 100644
--- a/.github/workflows/generic-pr/label-config.yml
+++ b/.github/workflows/generic-pr/label-config.yml
@@ -15,6 +15,15 @@
 # limitations under the License.
 #
 
+camel-3:
+  - base-branch:
+      - '^camel-3.x'
+
+camel-4:
+  - base-branch:
+      - 'main'
+      - '^camel-4.x'
+
 ci:
   - changed-files:
       - any-glob-to-any-file: