Test adding expiration config for queue messages

config/cdConfig.js

@@ -129,6 +129,7 @@ module.exports = {
       weights: { immediate: 3, soon: 2, normal: 3, later: 2 },
       connectionString: config.get('CRAWLER_QUEUE_AZURE_CONNECTION_STRING') || cd_azblob.connection,
       queueName: config.get('CRAWLER_QUEUE_PREFIX') || 'cdcrawlerdev',
+      messageTimeToLive: config.get('CRAWLER_QUEUE_MESSAGE_EXPIRATION') || 60 * 60 * 24 * 7, // Default to 7 days
       visibilityTimeout: 8 * 60 * 60, // 8 hours
       visibilityTimeout_remainLocal: fetchedCacheTtlSeconds,
       maxDequeueCount: 5,

ghcrawler/providers/queuing/storageQueue.js

@@ -37,6 +37,7 @@ class StorageQueue {
         qlimit(this.options.parallelPush || 1)(request => {
           const body = JSON.stringify(request)
           return new Promise((resolve, reject) => {
+            this.logger.info(`Sending createMessage to ${this.queueName} with options ${JSON.stringify(option)}`)
             this.client.createMessage(this.queueName, body, option, (error, queueMessageResult) => {
               if (error) {
                 return reject(error)

test/unit/providers/queuing/storageQueueManagerTests.js

@@ -0,0 +1,25 @@
+const sinon = require('sinon')
+const { expect } = require('chai')
+const StorageQueueManager = require('../../../../ghcrawler/providers/queuing/storageQueueManager')
+const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue')
+
+describe('StorageQueueManager', () => {
+  let manager, clientStub
+
+  beforeEach(() => {
+    clientStub = {
+      createMessage: sinon.stub().yields(null, {})
+    }
+    manager = new StorageQueueManager('connectionString', { messageTimeToLive: 3600 }) // 1 hour expiration
+    manager.client = clientStub
+  })
+
+  it('should create a queue with the correct expiration', async () => {
+    const queue = manager.createQueue('testQueue')
+    await queue.push({ body: 'test message' })
+
+    expect(clientStub.createMessage.calledOnce).to.be.true
+    const args = clientStub.createMessage.getCall(0).args
+    expect(args[2].messageTimeToLive).to.equal(3600)
+  })
+})

test/unit/providers/queuing/storageQueueTests.js

@@ -0,0 +1,22 @@
+const sinon = require('sinon')
+const { expect } = require('chai')
+const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue')
+
+describe('StorageQueue', () => {
+  let queue, clientStub
+
+  beforeEach(() => {
+    clientStub = {
+      createMessage: sinon.stub().yields(null, {})
+    }
+    queue = new StorageQueue(clientStub, 'testQueue', 'testQueueName', message => message, { messageTimeToLive: 3600 }) // 1 hour expiration
+  })
+
+  it('should add messages with the correct expiration', async () => {
+    await queue.push({ body: 'test message' })
+
+    expect(clientStub.createMessage.calledOnce).to.be.true
+    const args = clientStub.createMessage.getCall(0).args
+    expect(args[2].messageTimeToLive).to.equal(3600)
+  })
+})