chore(sessions): Add worker to cleanup older sessions. (#14319)

* Add worker to cleanup older sessions. * chore: charts update dirty files * Fix logging. * chore: nx format:write update dirty files * Error handling. * Remove env var. * chore: charts update dirty files * Rename files. * Use shared test setup utility. * No checking for rows to delete. * Use logger. * Update apps/services/sessions/src/app/cleanup/cleanup-worker.service.spec.ts Co-authored-by: Sævar Már Atlason <[email protected]> * Add comment explaining assertion. * chore: charts update dirty files --------- Co-authored-by: andes-it <[email protected]> Co-authored-by: Sævar Már Atlason <[email protected]> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
island-is · Apr 24, 2024 · 31c5144 · 31c5144
1 parent bb04cf6
commit 31c5144
Show file tree

Hide file tree

Showing 11 changed files with 398 additions and 2 deletions.
diff --git a/apps/services/sessions/infra/sessions.ts b/apps/services/sessions/infra/sessions.ts
@@ -1,4 +1,3 @@
-import { PersistentVolumeClaim } from '../../../../infra/src/dsl/types/input-types'
 import { service, ServiceBuilder } from '../../../../infra/src/dsl/dsl'
 
 const namespace = 'services-sessions'
@@ -84,3 +83,36 @@ export const workerSetup = (): ServiceBuilder<'services-sessions-worker'> =>
       },
       REDIS_USE_SSL: 'true',
     })
+
+const cleanupId = 'services-sessions-cleanup'
+// run daily at 3am
+const schedule = '0 3 * * *'
+
+export const cleanupSetup = (): ServiceBuilder<typeof cleanupId> =>
+  service(cleanupId)
+    .namespace(namespace)
+    .image(imageName)
+    .command('node')
+    .args('main.js', '--job=cleanup')
+    .resources({
+      limits: {
+        cpu: '400m',
+        memory: '512Mi',
+      },
+      requests: {
+        cpu: '100m',
+        memory: '256Mi',
+      },
+    })
+    .db()
+    .extraAttributes({
+      dev: {
+        schedule,
+      },
+      staging: {
+        schedule,
+      },
+      prod: {
+        schedule,
+      },
+    })
diff --git a/apps/services/sessions/project.json b/apps/services/sessions/project.json
@@ -75,6 +75,16 @@
         "args": ["--job", "worker"]
       }
     },
+    "cleanup": {
+      "executor": "@nx/js:node",
+      "options": {
+        "buildTarget": "services-sessions:build",
+        "buildTargetOptions": {
+          "outputPath": "dist/apps/services/sessions/cleanup"
+        },
+        "args": ["--job", "cleanup"]
+      }
+    },
     "dev-services": {
       "executor": "nx:run-commands",
       "options": {

diff --git a/apps/services/sessions/src/app/cleanup/cleanup-worker.module.ts b/apps/services/sessions/src/app/cleanup/cleanup-worker.module.ts
@@ -0,0 +1,20 @@
+import { Module } from '@nestjs/common'
+import { SequelizeModule } from '@nestjs/sequelize'
+
+import { LoggingModule } from '@island.is/logging'
+
+import { SequelizeConfigService } from '../../sequelizeConfig.service'
+import { Session } from '../sessions/session.model'
+import { SessionsCleanupService } from './cleanup-worker.service'
+
+@Module({
+  imports: [
+    LoggingModule,
+    SequelizeModule.forRootAsync({
+      useClass: SequelizeConfigService,
+    }),
+    SequelizeModule.forFeature([Session]),
+  ],
+  providers: [SessionsCleanupService],
+})
+export class SessionsCleanupWorkerModule {}
diff --git a/apps/services/sessions/src/app/cleanup/cleanup-worker.service.spec.ts b/apps/services/sessions/src/app/cleanup/cleanup-worker.service.spec.ts
@@ -0,0 +1,65 @@
+import { createNationalId } from '@island.is/testing/fixtures'
+import { setupAppWithoutAuth, TestApp } from '@island.is/testing/nest'
+
+import { FixtureFactory } from '../../../test/fixture.factory'
+import { SequelizeConfigService } from '../../sequelizeConfig.service'
+import { Session } from '../sessions/session.model'
+import { SessionsCleanupWorkerModule } from './cleanup-worker.module'
+import { SessionsCleanupService } from './cleanup-worker.service'
+
+describe('SessionsService', () => {
+  let app: TestApp
+  let sessionsCleanupService: SessionsCleanupService
+  let factory: FixtureFactory
+
+  beforeAll(async () => {
+    app = await setupAppWithoutAuth({
+      AppModule: SessionsCleanupWorkerModule,
+      SequelizeConfigService,
+      dbType: 'sqlite',
+    })
+    factory = new FixtureFactory(app)
+
+    sessionsCleanupService = app.get(SessionsCleanupService)
+  })
+
+  beforeEach(async () => {
+    await factory.get(Session).destroy({
+      where: {},
+      cascade: true,
+      truncate: true,
+      force: true,
+    })
+  })
+
+  afterAll(async () => {
+    await app?.cleanUp()
+  })
+
+  it('should remove old enough session records', async () => {
+    // Arrange
+
+    // Create sessions that should be deleted
+    await factory.createDateSessions(
+      createNationalId('person'),
+      new Date('2023-01-01'),
+      new Date('2023-02-01'),
+    )
+
+    // Create sessions that should remain
+    await factory.createDateSessions(
+      createNationalId('person'),
+      new Date('3023-01-01'),
+      new Date('3023-02-01'),
+    )
+
+    // Act
+    await sessionsCleanupService.run()
+
+    // Assert
+    const sessions = await factory.get(Session).findAll()
+    expect(sessions).toHaveLength(5)
+    // Check that all remaining sessions are newer than the cutoff date
+    expect(sessions.every((s) => s.timestamp > new Date('2023-02-01')))
+  })
+})
diff --git a/apps/services/sessions/src/app/cleanup/cleanup-worker.service.ts b/apps/services/sessions/src/app/cleanup/cleanup-worker.service.ts
@@ -0,0 +1,50 @@
+import { Inject } from '@nestjs/common'
+import { InjectModel } from '@nestjs/sequelize'
+import subMonths from 'date-fns/subMonths'
+import { Op } from 'sequelize'
+
+import { LOGGER_PROVIDER } from '@island.is/logging'
+
+import { Session } from '../sessions/session.model'
+
+import type { Logger } from '@island.is/logging'
+
+const RETENTION_IN_MONTHS = 12
+export class SessionsCleanupService {
+  constructor(
+    @Inject(LOGGER_PROVIDER)
+    private readonly logger: Logger,
+    @InjectModel(Session)
+    private readonly sessionModel: typeof Session,
+  ) {}
+
+  public async run() {
+    const timer = this.logger.startTimer()
+    this.logger.info('Worker starting...')
+
+    await this.deleteOlderSessions()
+
+    this.logger.info('Worker finished.')
+    timer.done()
+  }
+
+  private async deleteOlderSessions() {
+    this.logger.info(`Deleting...`)
+
+    const filter = {
+      where: {
+        timestamp: {
+          [Op.lt]: subMonths(new Date(), RETENTION_IN_MONTHS),
+        },
+      },
+    }
+
+    const affectedRows: number = await this.sessionModel.destroy(filter)
+
+    if (affectedRows > 0) {
+      this.logger.info(`Finished deleting ${affectedRows} rows.`)
+    } else {
+      this.logger.info(`No rows found to delete.`)
+    }
+  }
+}
diff --git a/apps/services/sessions/src/app/cleanup/cleanup-worker.ts b/apps/services/sessions/src/app/cleanup/cleanup-worker.ts
@@ -0,0 +1,23 @@
+import { NestFactory } from '@nestjs/core'
+
+import { logger } from '@island.is/logging'
+
+import { SessionsCleanupWorkerModule } from './cleanup-worker.module'
+import { SessionsCleanupService } from './cleanup-worker.service'
+
+export const worker = async () => {
+  try {
+    logger.info('Sessions cleanup worker started.')
+    const app = await NestFactory.createApplicationContext(
+      SessionsCleanupWorkerModule,
+    )
+    app.enableShutdownHooks()
+    await app.get(SessionsCleanupService).run()
+    await app.close()
+    logger.info('Sessions cleanup worker finished successfully.')
+    process.exit(0)
+  } catch (error) {
+    logger.error('Sessions cleanup worker encountered an error:', error)
+    process.exit(1)
+  }
+}
diff --git a/apps/services/sessions/src/main.ts b/apps/services/sessions/src/main.ts
@@ -1,5 +1,7 @@
-import { bootstrap, processJob } from '@island.is/infra-nest-server'
 import ip3country from 'ip3country'
+
+import { bootstrap, processJob } from '@island.is/infra-nest-server'
+
 import { AppModule } from './app/app.module'
 import { WorkerModule } from './app/worker/worker.module'
 import { environment } from './environments'
@@ -16,6 +18,12 @@ if (job === 'worker') {
     name: 'sessions-worker',
     beforeAppInit,
   })
+} else if (job === 'cleanup') {
+  import('./app/cleanup/cleanup-worker')
+    .then((app) => app.worker())
+    .catch((error) => {
+      console.error('Failed to import or execute the cleanup worker:', error)
+    })
 } else {
   bootstrap({
     appModule: AppModule,

diff --git a/charts/islandis/values.dev.yaml b/charts/islandis/values.dev.yaml
@@ -2481,6 +2481,67 @@ services-sessions:
   securityContext:
     allowPrivilegeEscalation: false
     privileged: false
+services-sessions-cleanup:
+  args:
+    - 'main.js'
+    - '--job=cleanup'
+  command:
+    - 'node'
+  enabled: true
+  env:
+    DB_HOST: 'postgres-applications.internal'
+    DB_NAME: 'services_sessions_cleanup'
+    DB_REPLICAS_HOST: 'postgres-applications-reader.internal'
+    DB_USER: 'services_sessions_cleanup'
+    LOG_LEVEL: 'info'
+    NODE_OPTIONS: '--max-old-space-size=460'
+    SERVERSIDE_FEATURES_ON: ''
+  grantNamespaces:
+    - 'nginx-ingress-internal'
+    - 'islandis'
+    - 'identity-server'
+  grantNamespacesEnabled: true
+  healthCheck:
+    liveness:
+      initialDelaySeconds: 3
+      path: '/'
+      timeoutSeconds: 3
+    readiness:
+      initialDelaySeconds: 3
+      path: '/'
+      timeoutSeconds: 3
+  hpa:
+    scaling:
+      metric:
+        cpuAverageUtilization: 90
+        nginxRequestsIrate: 5
+      replicas:
+        max: 3
+        min: 1
+  image:
+    repository: '821090935708.dkr.ecr.eu-west-1.amazonaws.com/services-sessions'
+  namespace: 'services-sessions'
+  podDisruptionBudget:
+    maxUnavailable: 1
+  pvcs: []
+  replicaCount:
+    default: 1
+    max: 3
+    min: 1
+  resources:
+    limits:
+      cpu: '400m'
+      memory: '512Mi'
+    requests:
+      cpu: '100m'
+      memory: '256Mi'
+  schedule: '0 3 * * *'
+  secrets:
+    CONFIGCAT_SDK_KEY: '/k8s/configcat/CONFIGCAT_SDK_KEY'
+    DB_PASS: '/k8s/services-sessions-cleanup/DB_PASSWORD'
+  securityContext:
+    allowPrivilegeEscalation: false
+    privileged: false
 services-sessions-worker:
   args:
     - 'main.js'

diff --git a/charts/islandis/values.prod.yaml b/charts/islandis/values.prod.yaml
@@ -2281,6 +2281,67 @@ services-sessions:
   securityContext:
     allowPrivilegeEscalation: false
     privileged: false
+services-sessions-cleanup:
+  args:
+    - 'main.js'
+    - '--job=cleanup'
+  command:
+    - 'node'
+  enabled: true
+  env:
+    DB_HOST: 'postgres-applications.internal'
+    DB_NAME: 'services_sessions_cleanup'
+    DB_REPLICAS_HOST: 'postgres-applications.internal'
+    DB_USER: 'services_sessions_cleanup'
+    LOG_LEVEL: 'info'
+    NODE_OPTIONS: '--max-old-space-size=460'
+    SERVERSIDE_FEATURES_ON: 'driving-license-use-v1-endpoint-for-v2-comms'
+  grantNamespaces:
+    - 'nginx-ingress-internal'
+    - 'islandis'
+    - 'identity-server'
+  grantNamespacesEnabled: true
+  healthCheck:
+    liveness:
+      initialDelaySeconds: 3
+      path: '/'
+      timeoutSeconds: 3
+    readiness:
+      initialDelaySeconds: 3
+      path: '/'
+      timeoutSeconds: 3
+  hpa:
+    scaling:
+      metric:
+        cpuAverageUtilization: 90
+        nginxRequestsIrate: 5
+      replicas:
+        max: 10
+        min: 3
+  image:
+    repository: '821090935708.dkr.ecr.eu-west-1.amazonaws.com/services-sessions'
+  namespace: 'services-sessions'
+  podDisruptionBudget:
+    maxUnavailable: 1
+  pvcs: []
+  replicaCount:
+    default: 3
+    max: 10
+    min: 3
+  resources:
+    limits:
+      cpu: '400m'
+      memory: '512Mi'
+    requests:
+      cpu: '100m'
+      memory: '256Mi'
+  schedule: '0 3 * * *'
+  secrets:
+    CONFIGCAT_SDK_KEY: '/k8s/configcat/CONFIGCAT_SDK_KEY'
+    DB_PASS: '/k8s/services-sessions-cleanup/DB_PASSWORD'
+  securityContext:
+    allowPrivilegeEscalation: false
+    privileged: false
 services-sessions-worker:
   args:
     - 'main.js'