V3 (#17)

* Intrapipeline Queueing (#16) * documentation: update comments * feat: add internal queue * Job, Command, Renames / Recontextulization (#18) * feat: renames * feat: add ability to ignore specific workflows * fix: formatting * fix: shellcheck * fix: job name * fix: shellcheck * Validate Queue Behavior (#19) * feat: ensure we can validate the correct ordering of jobs * fix: rename to the correct pipeline to ignore * debugging * fix: debug * walk back validation * feat: update documentation * Update src/jobs/global-queue.yml Co-authored-by: Andy Horner <[email protected]> * Update src/commands/global_block.yml Co-authored-by: Andy Horner <[email protected]> --------- Co-authored-by: Andy Horner <[email protected]>
promiseofcake · Apr 16, 2024 · 96876d5 · 96876d5
1 parent 369a5f8
commit 96876d5
Show file tree

Hide file tree

Showing 16 changed files with 409 additions and 161 deletions.
diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml
@@ -7,25 +7,51 @@ filters: &filters
   tags:
     only: /.*/
 
+jobs:
+  sleep:
+    docker:
+      - image: cimg/base:edge
+    parameters:
+      time:
+        type: integer
+    steps:
+      - run: echo "sleeping << parameters.time >>s" &&  sleep << parameters.time >>
+
 workflows:
-  test-deploy:
+  test-pipeline-a:
     jobs:
-      - orb-tools/pack:
-          filters: *filters
-      - workflow-queue/queue:
+      - workflow-queue/pipeline-queue:
+          context: orb-publishing
+          debug: true
+  test-pipeline-b:
+    jobs:
+      - sleep:
+          time: 5
+  test-pipeline-c:
+    jobs:
+      - sleep:
+          time: 20
+
+  test-global-queue:
+    jobs:
+      - workflow-queue/global-queue:
+          context: orb-publishing
+          debug: true
+          ignored-workflows: "test-pipeline-a" # given we are in one pipeline, don't block on ourselves
           filters:
             tags:
               ignore: /.*/
-          context: orb-publishing
-          requires: [orb-tools/pack]
+  test-deploy:
+    jobs:
+      - orb-tools/pack:
+          filters: *filters
       - orb-tools/publish:
+          context: orb-publishing
           orb-name: promiseofcake/workflow-queue
           vcs-type: << pipeline.project.type >>
           pub-type: production
           requires:
             - orb-tools/pack
-            - workflow-queue/queue
-          context: orb-publishing
           filters:
             branches:
               ignore: /.*/

diff --git a/Makefile b/Makefile
@@ -0,0 +1,5 @@
+test-global-queue:
+	./src/scripts/test.sh global-queue.sh
+
+test-pipeline-queue:
+	./src/scripts/test.sh pipeline-queue.sh
diff --git a/README.md b/README.md
@@ -4,9 +4,11 @@
 
 ## Introduction
 
-Forked from <https://github.com/eddiewebb/circleci-queue> and updated to reduce the use-cases, and migrate to the CircleCI V2 API
+Originally forked from <https://github.com/eddiewebb/circleci-queue> and updated to reduce the use-cases, and migrate to the CircleCI V2 API
 
-The purpose of this Orb is to add a concept of a queue to specific branch workflow tasks in CircleCi. The main use-case is to isolate a set of changes to ensure that one set of a thing is running at one time. Think of smoke-tests against a nonproduction environment as a promotion gate.
+The purpose of this Orb is to add a concept of a queue to specific branch's workflow tasks in CircleCi. The main use-case is to isolate a set of changes to ensure that one set of a thing is running at one time. Think of smoke-tests against a nonproduction environment as a promotion gate.
+
+Additional use-cases are for queueing workflows within a given pipeline (a feature missing today from CircleCi).
 
 ## Configuration Requirements
 

diff --git a/src/README.md b/src/README.md
diff --git a/src/commands/block_execution.yml b/src/commands/block_execution.yml
diff --git a/src/commands/global_block.yml b/src/commands/global_block.yml
@@ -0,0 +1,48 @@
+description: >
+  When used, this command blocks the assigned workflow from running until all previous (global) workflows have completed.
+  This ensures only one instance of a given workflow is running at a time across all defined branches.
+parameters:
+  debug:
+    type: boolean
+    default: false
+    description: "When enabled, additional debug logging with be output."
+  time:
+    type: string
+    default: "10"
+    description: "Number of minutes to wait for a lock before giving up."
+  dont-quit:
+    type: boolean
+    default: false
+    description: "If true, forces the job through once time expires instead of failing."
+  only-on-branch:
+    type: string
+    default: "*"
+    description: "Only queue on specified branch"
+  confidence:
+    type: string
+    default: "1"
+    description: >
+      Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending",
+      but not yet active. This number indicates the threshold for API returning no previous pending pipelines.
+      Default is `1` confirmation, increase if you see issues.
+  ignored-workflows:
+    type: string
+    default: ""
+    description: Comma separated list of workflow names to ignore as blocking workflows for the global queue.
+  include-on-hold:
+    type: boolean
+    default: false
+    description: Consider `on-hold` workflows waiting for approval as running and include them in the queue.
+
+steps:
+  - run:
+      name: Block execution until the current workflow is at the front of the line
+      environment:
+        CONFIG_DEBUG_ENABLED: "<< parameters.debug >>"
+        CONFIG_TIME: "<< parameters.time >>"
+        CONFIG_DONT_QUIT: "<< parameters.dont-quit >>"
+        CONFIG_ONLY_ON_BRANCH: "<< parameters.only-on-branch >>"
+        CONFIG_CONFIDENCE: "<< parameters.confidence >>"
+        CONFIG_IGNORED_WORKFLOWS: "<< parameters.ignored-workflows >>"
+        CONFIG_INCLUDE_ON_HOLD: " << parameters.include-on-hold >>"
+      command: <<include(scripts/global-queue.sh)>>
diff --git a/src/commands/pipeline_block.yml b/src/commands/pipeline_block.yml
@@ -0,0 +1,23 @@
+description: >
+  This command blocks execution of a workflow within the context of a given pipeline.
+
+parameters:
+  debug:
+    type: boolean
+    default: false
+    description: "When enabled, additional debug logging with be output."
+  confidence:
+    type: string
+    default: "1"
+    description: >
+      Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending",
+      but not yet active. This number indicates the threshold for API returning no previous pending pipelines.
+      Default is `1` confirmation, increase if you see issues.
+
+steps:
+  - run:
+      name: Blocking execution until the current workflow is the last to run in the given pipeline.
+      environment:
+        CONFIG_DEBUG_ENABLED: "<< parameters.debug >>"
+        CONFIG_CONFIDENCE: "<< parameters.confidence >>"
+      command: <<include(scripts/pipeline-queue.sh)>>
diff --git a/src/examples/example.yml b/src/examples/example.yml
@@ -3,7 +3,7 @@ description: >
 usage:
   version: 2.1
   orbs:
-    workflow-queue: promiseofcake/workflow-queue@1
+    workflow-queue: promiseofcake/workflow-queue@2
   workflows:
     example:
       jobs:

diff --git a/src/jobs/global-queue.yml b/src/jobs/global-queue.yml
@@ -0,0 +1,49 @@
+description: >
+  This job ensures only a single defined global workflow is running at a given point in time.
+
+docker:
+  - image: cimg/base:stable
+resource_class: small
+
+parameters:
+  debug:
+    type: boolean
+    default: false
+    description: "When enabled, additional debug logging with be output."
+  time:
+    type: string
+    default: "10"
+    description: "Number of minutes to wait for a lock before giving up."
+  dont-quit:
+    type: boolean
+    default: false
+    description: "If true, forces the job through once time expires instead of failing."
+  only-on-branch:
+    type: string
+    default: "*"
+    description: "Only queue on specified branch"
+  confidence:
+    type: string
+    default: "1"
+    description: >
+      Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending",
+      but not yet active. This number indicates the threshold for API returning no previous pending pipelines.
+      Default is `1` confirmation, increase if you see issues.
+  ignored-workflows:
+    type: string
+    default: ""
+    description: Comma separated list of workflow names to ignore as blocking workflows for the global queue.
+  include-on-hold:
+    type: boolean
+    default: false
+    description: Consider `on-hold` workflows waiting for approval as running and include them in the queue.
+
+steps:
+  - global_block:
+      debug: << parameters.debug >>
+      time: << parameters.time >>
+      dont-quit: << parameters.dont-quit >>
+      only-on-branch: << parameters.only-on-branch >>
+      confidence: << parameters.confidence >>
+      ignored-workflows: << parameters.ignored-workflows >>
+      include-on-hold: << parameters.include-on-hold >>
diff --git a/src/jobs/pipeline-queue.yml b/src/jobs/pipeline-queue.yml
@@ -0,0 +1,24 @@
+description: >
+  This job prevents a workflow within a given pipeline from running until all previous workflows have completed.
+
+docker:
+  - image: cimg/base:stable
+resource_class: small
+
+parameters:
+  debug:
+    type: boolean
+    default: false
+    description: "If enabled, DEBUG messages will be logged."
+  confidence:
+    type: string
+    default: "1"
+    description: >
+      Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending",
+      but not yet active. This number indicates the threshold for API returning no previous pending pipelines.
+      Default is one confirmation, increase if you see issues.
+
+steps:
+  - pipeline_block:
+      debug: <<parameters.debug>>
+      confidence: <<parameters.confidence>>
diff --git a/src/jobs/queue.yml b/src/jobs/queue.yml