Add option to increase GHA runner disk space

.github/workflows/integration-test.yml

@@ -1,6 +1,27 @@
 name: pi-gen-action-integration-test
 on:
   workflow_dispatch:
+    inputs:
+      enable-noobs:
+        description: Enable NOOBS
+        default: true
+        required: false
+        type: boolean
+      compression-level:
+        description: Image compression level
+        default: 1
+        required: false
+        type: number
+      increase-runner-disk:
+        description: Increase runner root disk size
+        required: false
+        default: true
+        type: boolean
+      full-image:
+        description: Build all stages
+        default: false
+        required: false
+        type: boolean
   push:
     paths-ignore:
       - '**.md'
@@ -26,7 +47,7 @@ jobs:
 
   integration-test:
     runs-on: ubuntu-latest
-    if: github.ref_name == 'master' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'test'))
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'test')
 
     steps:
       - name: Check out repository
@@ -45,11 +66,11 @@ jobs:
         id: build
         with:
           image-name: integration-test
-          stage-list: stage0 stage1 stage2 ./__test__/it-test-stage
+          stage-list: stage0 stage1 stage2 ./__test__/it-test-stage ${{ inputs.full-image && 'stage3 stage4 stage5' }}
           verbose-output: true
-          enable-noobs: true
+          enable-noobs: ${{ github.event_name != 'workflow_dispatch' || inputs.enable-noobs }}
           compression: xz
-          compression-level: 1
+          compression-level: ${{ github.event_name == 'workflow_dispatch' && inputs.compression-level || 1 }}
           locale: ${{ env.CONFIG_LOCALE }}
           hostname: ${{ env.CONFIG_HOSTNAME }}
           keyboard-keymap: de
@@ -59,6 +80,7 @@ jobs:
           wpa-password: '1234567890'
           timezone: ${{ env.CONFIG_TIMEZONE }}
           pubkey-ssh-first-user: ${{ env.CONFIG_PUBLIC_KEY }}
+          increase-runner-disk-size: ${{ github.event_name != 'workflow_dispatch' || inputs.increase-runner-disk }}
 
       - name: List working directory
         run: tree 
@@ -81,6 +103,9 @@ jobs:
           test "$(cat ${ROOTFS_DIR}/etc/timezone)" = "$CONFIG_TIMEZONE"
           test "$(sudo cat ${ROOTFS_DIR}/home/${CONFIG_USERNAME}/.ssh/authorized_keys)" = "$CONFIG_PUBLIC_KEY"
 
+      - run: df -h
+        if: always()
+
       - name: Remove test label from PR (if set)
         uses: actions-ecosystem/action-remove-labels@v1
         if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'test') }}

README.md

@@ -109,6 +109,14 @@ tries to make sure the stage is respected and its changes are included in the fi
     # Final image name.
     image-name: ''
 
+    # Enabling this option will remove plenty of components from the GitHub Actions 
+    # runner that are not mandatory pre-requisites for a (vanilla) pi-gen build. This 
+    # shall increase the available disk space so that also large images can be 
+    # compiled on a free GHA runner (benchmark is the full image including a desktop 
+    # environment). If any packages are missing during the build consider adding them 
+    # to the `extra-host-dependencies` list.
+    increase-runner-disk-size: false
+
     # Default keyboard keymap.
     keyboard-keymap: gb
 
@@ -188,6 +196,7 @@ tries to make sure the stage is respected and its changes are included in the fi
 - [Enable detailed output from `pi-gen` build](#enable-detailed-output-from-pi-gen-build)
 - [Upload final image as artifact](#upload-final-image-as-artifact)
 - [Modify `pi-gen` internal stages](#modify-pi-gen-internal-stages)
+- [Increase GitHub Actions runner disk space](#increase-github-actions-runner-disk-space)
 
 ### Install NodeJS from Nodesource in the target image
 ```yaml
@@ -202,7 +211,7 @@ jobs:
           cat > test-stage/package-test/00-run-chroot.sh <<-EOF
           #!/bin/bash
           apt-get install -y curl
-          curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
+          curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
           EOF
           } &&
           chmod +x test-stage/package-test/00-run-chroot.sh &&
@@ -279,6 +288,32 @@ jobs:
           pi-gen-dir: ${{ inputs.custom-pi-gen-dir }}
 ```
 
+### Increase GitHub Actions runner disk space
+
+When building large images containing plenty of additional software (or maybe just the full
+image including a desktop environment) you may hit space boundaries on the (free and public)
+GitHub Actions runners. 
+
+There is however a workaround to increase this disk space by removing components you do not
+need for your build on the runner. The implemented mechanism is heavily inspired by
+[easimon/maximize-build-space](https://github.com/easimon/maximize-build-space) but focuses
+on providing more root disk space where the Docker daemon runs the build (since this action
+runs the `pi-gen` build always inside a container). 
+
+From current experience, this will reclaim between 25 and 30 GB of additional disk space
+
+```
+jobs:
+  pi-gen-with-larger-disk-space:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: usimd/pi-gen-action@v1
+        with:
+          image-name: test
+          stage-list: stage0 stage1 stage2 stage3 stage4 stage5
+          increase-runner-disk-size: true
+```
+
 ## License
 
 The scripts and documentation in this project are released under the [MIT License](LICENSE)

__test__/actions.test.ts

@@ -3,6 +3,7 @@ import {DEFAULT_CONFIG} from '../src/pi-gen-config'
 import * as actions from '../src/actions'
 import {removeContainer} from '../src/remove-container'
 import {build} from '../src/build'
+import {removeRunnerComponents} from '../src/increase-runner-disk-size'
 
 jest.mock('../src/configure', () => ({
   configure: jest.fn().mockReturnValue(DEFAULT_CONFIG)
@@ -11,14 +12,35 @@ jest.mock('../src/install-dependencies')
 jest.mock('../src/build')
 jest.mock('../src/clone-pigen')
 jest.mock('../src/remove-container')
+jest.mock('../src/increase-runner-disk-size')
 
 describe('Actions', () => {
+  const OLD_ENV = process.env
+
+  beforeEach(() => {
+    jest.resetModules()
+    process.env = {...OLD_ENV}
+  })
+
+  afterAll(() => {
+    process.env = OLD_ENV
+  })
+
+  it('should only increase disk space if requested', async () => {
+    jest.spyOn(core, 'getBooleanInput').mockReturnValueOnce(true)
+
+    await actions.piGen()
+
+    expect(removeRunnerComponents).toHaveBeenCalled()
+  })
+
   it('does not run build function twice but invokes cleanup', async () => {
     jest
       .spyOn(core, 'getState')
       .mockReturnValueOnce('')
       .mockReturnValueOnce('true')
       .mockReturnValueOnce('true')
+    process.env['INPUT_INCREASE-RUNNER-DISK-SIZE'] = 'false'
 
     // expect build here
     await actions.run()
@@ -29,6 +51,34 @@ describe('Actions', () => {
     expect(removeContainer).toHaveBeenCalledTimes(1)
   })
 
+  const errorMessage = 'any error'
+  it.each([new Error(errorMessage), errorMessage])(
+    'should catch errors thrown during build and set build safely as failed',
+    async error => {
+      const errorMessage = 'any error'
+      jest.spyOn(core, 'getInput').mockImplementation((name, options) => {
+        throw error
+      })
+      jest.spyOn(core, 'setFailed')
+
+      await expect(actions.piGen()).resolves.not.toThrow()
+      expect(core.setFailed).toHaveBeenLastCalledWith(errorMessage)
+    }
+  )
+
+  it.each([new Error(errorMessage), errorMessage])(
+    'should gracefully catch errors thrown during cleanup and emit a warning message',
+    async error => {
+      jest.spyOn(core, 'getState').mockImplementation(name => {
+        throw error
+      })
+      jest.spyOn(core, 'warning')
+
+      await expect(actions.cleanup()).resolves.not.toThrow()
+      expect(core.warning).toHaveBeenLastCalledWith(errorMessage)
+    }
+  )
+
   describe('cleanup', () => {
     it.each(['', 'true'])(
       'tries to remove container only if build has started = %s',

__test__/increase-runner-disk-size.test.ts

@@ -0,0 +1,49 @@
+import * as exec from '@actions/exec'
+import {removeRunnerComponents} from '../src/increase-runner-disk-size'
+
+jest.mock('@actions/exec')
+
+describe('Increasing runner disk size', () => {
+  it('should prune Docker system, remove defined host paths and invoke apt', async () => {
+    jest
+      .spyOn(exec, 'getExecOutput')
+      .mockImplementation((commandLine, args, options) => {
+        if (commandLine === 'sh') {
+          return Promise.resolve({stdout: ' 12345 '} as exec.ExecOutput)
+        } else {
+          return Promise.resolve({} as exec.ExecOutput)
+        }
+      })
+    await removeRunnerComponents()
+
+    expect(exec.getExecOutput).toHaveBeenCalledWith(
+      'sudo',
+      expect.arrayContaining(['docker', 'system', 'prune']),
+      expect.anything()
+    )
+
+    expect(exec.getExecOutput).toHaveBeenCalledWith(
+      'sudo',
+      expect.arrayContaining(['rm', '-rf']),
+      expect.anything()
+    )
+
+    expect(exec.getExecOutput).toHaveBeenCalledWith(
+      'sudo',
+      expect.arrayContaining(['apt-get', 'autoremove']),
+      expect.anything()
+    )
+
+    expect(exec.getExecOutput).toHaveBeenCalledWith(
+      'sudo',
+      expect.arrayContaining(['apt-get', 'autoclean']),
+      expect.anything()
+    )
+
+    expect(exec.getExecOutput).toHaveBeenCalledWith(
+      'sudo',
+      expect.arrayContaining(['swapoff', '-a']),
+      expect.anything()
+    )
+  })
+})

action.yml

@@ -136,6 +136,14 @@ inputs:
       If your custom stage requires additional software or kernel modules to be loaded, add them here. Note that this is not meant to configure modules to be loaded in the target image.
     required: false
     default: ''
+  increase-runner-disk-size:
+    description: |
+      Enabling this option will remove plenty of components from the GitHub Actions runner that are not mandatory pre-requisites for a (vanilla) pi-gen build.
+      This shall increase the available disk space so that also large images can be compiled on a free GHA runner (benchmark is the full image including a 
+      desktop environment).
+      If any packages are missing during the build consider adding them to the `extra-host-dependencies` list.
+    required: false
+    default: false
   pi-gen-dir:
     description: Path where selected pi-gen ref will be checked out to. If the path does not yet exist, it will be created (including its parents).
     required: false

jest.config.js

@@ -3,10 +3,10 @@ module.exports = {
   collectCoverageFrom: ['src/**/*.ts', '!src/misc/update-readme.ts'],
   coverageThreshold: {
     global: {
-      statements: 97,
-      branches: 92,
+      statements: 98,
+      branches: 95,
       functions: 96,
-      lines: 97
+      lines: 98
     }
   },
   clearMocks: true,

src/actions.ts

@@ -4,6 +4,7 @@ import {installHostDependencies} from './install-dependencies'
 import {build} from './build'
 import {clonePigen} from './clone-pigen'
 import {removeContainer} from './remove-container'
+import {removeRunnerComponents} from './increase-runner-disk-size'
 
 const piGenBuildStartedState = 'pi-gen-build-started'
 
@@ -19,7 +20,18 @@ export async function piGen(): Promise<void> {
     const piGenRepo = core.getInput('pi-gen-repository')
     core.debug(`Using pi-gen repository ${piGenRepo}`)
 
+    const increaseRunnerDiskSize = core.getBooleanInput(
+      'increase-runner-disk-size'
+    )
+    core.debug(`Increase runner disk size: ${increaseRunnerDiskSize}`)
+
     const userConfig = await configure()
+
+    if (increaseRunnerDiskSize) {
+      core.info('Removing unused runner components to increase disk space')
+      await removeRunnerComponents()
+    }
+
     await clonePigen(piGenRepo, piGenDirectory, core.getInput('pi-gen-version'))
     await installHostDependencies(
       core.getInput('extra-host-dependencies'),