Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallelize host cleanup steps #143

Merged
merged 1 commit into from
Nov 15, 2024
Merged

Parallelize host cleanup steps #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions __test__/increase-runner-disk-size.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import * as exec from '@actions/exec'
import {removeRunnerComponents} from '../src/increase-runner-disk-size'
import * as core from '@actions/core'
import {
getExecOptions,
removeRunnerComponents
} from '../src/increase-runner-disk-size'

jest.mock('@actions/exec')

Expand All @@ -14,6 +18,8 @@ describe('Increasing runner disk size', () => {
return Promise.resolve({} as exec.ExecOutput)
}
})
jest.spyOn(core, 'getBooleanInput').mockReturnValueOnce(true)

await removeRunnerComponents()

expect(exec.getExecOutput).toHaveBeenCalledWith(
Expand All @@ -30,13 +36,11 @@ describe('Increasing runner disk size', () => {

expect(exec.getExecOutput).toHaveBeenCalledWith(
'sudo',
expect.arrayContaining(['apt-get', 'autoremove']),
expect.anything()
)

expect(exec.getExecOutput).toHaveBeenCalledWith(
'sudo',
expect.arrayContaining(['apt-get', 'autoclean']),
expect.arrayContaining([
'sh',
'-c',
'apt-get autoremove && apt-get autoclean'
]),
expect.anything()
)

Expand All @@ -46,4 +50,14 @@ describe('Increasing runner disk size', () => {
expect.anything()
)
})

it('should invoke execution log callbacks only when verbose', async () => {
jest.spyOn(core, 'info')
const opts = getExecOptions('docker-system-prune', true)

opts.listeners?.stdline('test')
opts.listeners?.errline('test')

expect(core.info).toHaveBeenCalledTimes(2)
})
})
3 changes: 1 addition & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@
"author": "Simon Domke",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"lint": "eslint src/**/*.ts",
"package": "ncc build src/main.ts -m --no-source-map-register --license licenses.txt",
"test": "jest",
"all": "npm run build && npm run format && npm run lint && npm test && npm run package && npm run update-readme",
"all": "npm run format && npm run lint && npm test && npm run package && npm run update-readme",
"format": "prettier --write '**/*.ts'",
"format-check": "prettier --check '**/*.ts'",
"update-readme": "ts-node src/misc/update-readme.ts"
Expand Down
6 changes: 4 additions & 2 deletions src/actions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@ export async function piGen(): Promise<void> {
const userConfig = await configure()

if (increaseRunnerDiskSize) {
core.info('Removing unused runner components to increase disk space')
await removeRunnerComponents()
await core.group(
'Removing runner components to increase disk build space',
() => removeRunnerComponents()
)
}

await clonePigen(piGenRepo, piGenDirectory, core.getInput('pi-gen-version'))
Expand Down
178 changes: 89 additions & 89 deletions src/increase-runner-disk-size.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,116 +1,116 @@
import * as exec from '@actions/exec'
import * as core from '@actions/core'
import * as colors from 'ansi-colors'

// See https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832
const HOST_PATHS_TO_REMOVE = [
'/opt/google/chrome',
'/opt/microsoft/msedge',
'/opt/microsoft/powershell',
'/opt/mssql-tools',
'/opt/hostedtoolcache',
'/opt/pipx',
'/usr/lib/mono',
'/usr/local/julia*',
'/usr/local/lib/android',
'/usr/local/lib/node_modules',
'/usr/local/share/chromium',
'/usr/local/share/powershell',
'/usr/share/dotnet',
'/usr/share/swift',
'/var/cache/snapd',
'/var/lib/snapd',
'/tmp/*',
'/usr/share/doc'
]

export async function removeRunnerComponents(): Promise<void> {
try {
core.startGroup('Removing runner components to increase disk build space')
const verbose = core.getBooleanInput('verbose-output')
const availableDiskSizeBeforeCleanup = await getAvailableDiskSize()
const actions = []

const availableDiskSizeBeforeCleanup = await getAvailableDiskSize()
core.debug(
`Available disk space before cleanup: ${availableDiskSizeBeforeCleanup / 1024 / 1024}G`
actions.push(
exec.getExecOutput(
'sudo',
['docker', 'system', 'prune', '--all', '--force'],
getExecOptions('docker-system-prune', verbose)
)
)

await exec
.getExecOutput(
'sudo',
['docker', 'system', 'prune', '--all', '--force'],
{
silent: true,
failOnStdErr: false,
ignoreReturnCode: true
}
)
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))

await exec
actions.push(
exec
.getExecOutput(
'sudo',
[
'sh',
'-c',
'snap list | sed 1d | cut -d" " -f1 | xargs -I{} snap remove {}'
],
{
silent: true,
failOnStdErr: false,
ignoreReturnCode: true
}
['swapoff', '-a'],
getExecOptions('swapoff', verbose)
)
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))

await exec
.getExecOutput('sudo', ['swapoff', '-a'], {
silent: true,
failOnStdErr: false,
ignoreReturnCode: true
})
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))

// See https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832
const hostPathsToRemove = [
'/opt/google/chrome',
'/opt/microsoft/msedge',
'/opt/microsoft/powershell',
'/opt/mssql-tools',
'/opt/hostedtoolcache',
'/opt/pipx',
'/usr/lib/mono',
'/usr/local/julia*',
'/usr/local/lib/android',
'/usr/local/lib/node_modules',
'/usr/local/share/chromium',
'/usr/local/share/powershell',
'/usr/share/dotnet',
'/usr/share/swift',
'/mnt/swapfile',
'/swapfile',
'/var/cache/snapd',
'/var/lib/snapd',
'/tmp/*',
'/usr/share/doc'
]

await exec
.getExecOutput('sudo', ['rm', '-rf', ...hostPathsToRemove], {
silent: true,
ignoreReturnCode: true,
failOnStdErr: false
.then(result => {
return exec.getExecOutput(
'sudo',
['rm', '-rf', '/mnt/swapfile', '/swapfile'],
getExecOptions('rm-swapfile', verbose)
)
})
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))
)

await exec
actions.push(
exec
.getExecOutput(
'sudo',
['apt', 'purge', 'snapd', 'php8*', 'r-base', 'imagemagick'],
{
silent: true,
ignoreReturnCode: true
}
['rm', '-rf', ...HOST_PATHS_TO_REMOVE],
getExecOptions('rm-host-paths', verbose)
)
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))
await exec
.getExecOutput('sudo', ['apt-get', 'autoremove'], {
silent: true,
ignoreReturnCode: true
})
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))
await exec
.getExecOutput('sudo', ['apt-get', 'autoclean'], {
silent: true,
ignoreReturnCode: true
.then((returnValue: exec.ExecOutput) => {
return exec
.getExecOutput(
'sudo',
['apt', 'purge', 'snapd', 'php8*', 'r-base', 'imagemagick'],
getExecOptions('apt-purge-packages', verbose)
)
.then((returnValue: exec.ExecOutput) => {
return exec.getExecOutput(
'sudo',
['sh', '-c', 'apt-get autoremove && apt-get autoclean'],
getExecOptions('apt-autoremove-autoclean', verbose)
)
})
})
.then((returnValue: exec.ExecOutput) => core.debug(returnValue.stdout))
)

return Promise.all(actions).then(async outputs => {
core.debug(
`Available disk space before cleanup: ${availableDiskSizeBeforeCleanup / 1024 / 1024}G`
)
const availableDiskSizeAfterCleanup = await getAvailableDiskSize()
core.debug(
`Available disk space after cleanup: ${availableDiskSizeAfterCleanup / 1024 / 1024}G`
)

core.info(
`Reclaimed runner disk space: ${((availableDiskSizeAfterCleanup - availableDiskSizeBeforeCleanup) / 1024 / 1024).toFixed(2)}G`
)
} finally {
core.endGroup()
})
}

const logPrefixColorTheme: Record<string, Function> = {
'docker-system-prune': colors.cyan,
swapoff: colors.redBright,
'rm-swapfile': colors.green,
'rm-host-paths': colors.magenta,
'apt-purge-packages': colors.yellow,
'apt-autoremove-autoclean': colors.blue
}

export function getExecOptions(logPrefix: string, verbose: boolean) {
return {
silent: true,
...(verbose && {
listeners: {
stdline: (line: string) =>
core.info(`${logPrefixColorTheme[logPrefix](logPrefix)}: ${line}`),
errline: (line: string) =>
core.info(`${logPrefixColorTheme[logPrefix](logPrefix)}: ${line}`)
}
})
}
}

Expand Down
Loading