Merge branch 'main' into lemusthelroy/ct-1160-point-build-system-to-n…

…ew-endpoint

.release-please-manifest.json

@@ -1,6 +1,6 @@
 {
   "packages/build-info": "7.13.2",
-  "packages/build": "29.41.6",
+  "packages/build": "29.42.1",
   "packages/edge-bundler": "12.0.1",
   "packages/cache-utils": "5.1.5",
   "packages/config": "20.12.5",

packages/build/CHANGELOG.md

@@ -105,6 +105,20 @@
   * dependencies
     * @netlify/config bumped from ^20.8.0 to ^20.8.1
 
+## [29.42.1](https://github.com/netlify/build/compare/build-v29.42.0...build-v29.42.1) (2024-05-22)
+
+
+### Bug Fixes
+
+* flush output from "Deploy site" step ([#5652](https://github.com/netlify/build/issues/5652)) ([076b4b0](https://github.com/netlify/build/commit/076b4b07c4595d943b0375edffe97eac22910cfd))
+
+## [29.42.0](https://github.com/netlify/build/compare/build-v29.41.6...build-v29.42.0) (2024-05-22)
+
+
+### Features
+
+* reduce build log verbosity ([#5643](https://github.com/netlify/build/issues/5643)) ([58def4f](https://github.com/netlify/build/commit/58def4f72eec4c1eb20cddceab0d8fd3c4420e45))
+
 ## [29.41.6](https://github.com/netlify/build/compare/build-v29.41.5...build-v29.41.6) (2024-05-22)
 
 

packages/build/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/build",
-  "version": "29.41.6",
+  "version": "29.42.1",
   "description": "Netlify build module",
   "type": "module",
   "exports": "./lib/index.js",
@@ -76,7 +76,7 @@
     "@netlify/functions-utils": "^5.2.57",
     "@netlify/git-utils": "^5.1.1",
     "@netlify/opentelemetry-utils": "^1.2.1",
-    "@netlify/plugins-list": "^6.79.0",
+    "@netlify/plugins-list": "^6.80.0",
     "@netlify/run-utils": "^5.1.1",
     "@netlify/zip-it-and-ship-it": "9.33.0",
     "@sindresorhus/slugify": "^2.0.0",

packages/build/src/core/feature_flags.ts

@@ -20,5 +20,6 @@ export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
   buildbot_zisi_system_log: false,
   edge_functions_cache_cli: false,
   edge_functions_system_logger: false,
+  netlify_build_reduced_output: false,
   netlify_build_updated_plugin_compatibility: false,
 }

packages/build/src/core/types.ts

@@ -52,6 +52,7 @@ type EventHandlers = {
     | {
         handler: NetlifyPlugin[K]
         description: string
+        quiet?: boolean
       }
 }
 

packages/build/src/install/missing.js

@@ -17,7 +17,7 @@ import { addExactDependencies } from './main.js'
 // their `package.json`.
 export const installMissingPlugins = async function ({ missingPlugins, autoPluginsDir, mode, logs }) {
   const packages = missingPlugins.map(getPackage)
-  logInstallMissingPlugins(logs, packages)
+  logInstallMissingPlugins(logs, missingPlugins, packages)
 
   if (packages.length === 0) {
     return

packages/build/src/log/logger.ts

@@ -4,10 +4,17 @@ import figures from 'figures'
 import indentString from 'indent-string'
 
 import { getHeader } from './header.js'
+import { OutputFlusher } from './output_flusher.js'
 import { serializeArray, serializeObject } from './serialize.js'
 import { THEME } from './theme.js'
 
-export type BufferedLogs = { stdout: string[]; stderr: string[] }
+export type Logs = BufferedLogs | StreamedLogs
+export type BufferedLogs = { stdout: string[]; stderr: string[]; outputFlusher?: OutputFlusher }
+export type StreamedLogs = { outputFlusher?: OutputFlusher }
+
+export const logsAreBuffered = (logs: Logs | undefined): logs is BufferedLogs => {
+  return logs !== undefined && 'stdout' in logs
+}
 
 const INDENT_SIZE = 2
 
@@ -35,7 +42,7 @@ export const getBufferLogs = (config: { buffer?: boolean }): BufferedLogs | unde
 // This should be used instead of `console.log()` as it allows us to instrument
 // how any build logs is being printed.
 export const log = function (
-  logs: BufferedLogs | undefined,
+  logs: Logs | undefined,
   string: string,
   config: { indent?: boolean; color?: (string: string) => string } = {},
 ) {
@@ -44,10 +51,12 @@ export const log = function (
   const stringB = String(stringA).replace(EMPTY_LINES_REGEXP, EMPTY_LINE)
   const stringC = color === undefined ? stringB : color(stringB)
 
-  if (logs !== undefined) {
-    // `logs` is a stateful variable
+  logs?.outputFlusher?.flush()
 
+  if (logsAreBuffered(logs)) {
+    // `logs` is a stateful variable
     logs.stdout.push(stringC)
+
     return
   }
 
@@ -178,3 +187,16 @@ export const getSystemLogger = function (
 
   return (...args) => fileDescriptor.write(`${reduceLogLines(args)}\n`)
 }
+
+export const addOutputGate = (logs: Logs, outputFlusher: OutputFlusher): Logs => {
+  if (logsAreBuffered(logs)) {
+    return {
+      ...logs,
+      outputFlusher,
+    }
+  }
+
+  return {
+    outputFlusher,
+  }
+}

packages/build/src/log/messages/core.ts

@@ -7,6 +7,7 @@ import { roundTimerToMillisecs } from '../../time/measure.js'
 import { ROOT_PACKAGE_JSON } from '../../utils/json.js'
 import { getLogHeaderFunc } from '../header_func.js'
 import { log, logMessage, logWarning, logHeader, logSubHeader, logWarningArray, BufferedLogs } from '../logger.js'
+import { OutputFlusher } from '../output_flusher.js'
 import { THEME } from '../theme.js'
 
 import { logConfigOnError } from './config.js'
@@ -29,13 +30,17 @@ export const logBuildError = function ({ error, netlifyConfig, logs, debug }) {
 
 export const logBuildSuccess = function (logs) {
   logHeader(logs, 'Netlify Build Complete')
-  logMessage(logs, '')
 }
 
-export const logTimer = function (logs, durationNs, timerName, systemLog) {
+export const logTimer = function (logs, durationNs, timerName, systemLog, outputFlusher?: OutputFlusher) {
   const durationMs = roundTimerToMillisecs(durationNs)
   const duration = prettyMs(durationMs)
-  log(logs, THEME.dimWords(`(${timerName} completed in ${duration})`))
+
+  if (!outputFlusher || outputFlusher.flushed) {
+    log(logs, '')
+    log(logs, THEME.dimWords(`(${timerName} completed in ${duration})`))
+  }
+
   systemLog(`Build step duration: ${timerName} completed in ${durationMs}ms`)
 }
 

packages/build/src/log/messages/install.js

@@ -1,20 +1,13 @@
 import { isRuntime } from '../../utils/runtime.js'
 import { log, logArray, logSubHeader } from '../logger.js'
 
-export const logInstallMissingPlugins = function (logs, packages) {
-  const runtimes = packages.filter((pkg) => isRuntime(pkg))
-  const plugins = packages.filter((pkg) => !isRuntime(pkg))
+export const logInstallMissingPlugins = function (logs, missingPlugins, packages) {
+  const plugins = missingPlugins.filter((pkg) => !isRuntime(pkg))
 
   if (plugins.length !== 0) {
     logSubHeader(logs, 'Installing plugins')
     logArray(logs, packages)
   }
-
-  if (runtimes.length !== 0) {
-    const [nextRuntime] = runtimes
-
-    logSubHeader(logs, `Using Next.js Runtime - v${nextRuntime.pluginPackageJson.version}`)
-  }
 }
 
 export const logInstallIntegrations = function (logs, integrations) {

packages/build/src/log/messages/steps.js

@@ -16,7 +16,3 @@ const getDescription = function ({ coreStepDescription, netlifyConfig, packageNa
 export const logBuildCommandStart = function (logs, buildCommand) {
   log(logs, THEME.highlightWords(`$ ${buildCommand}`))
 }
-
-export const logStepSuccess = function (logs) {
-  logMessage(logs, '')
-}

packages/build/src/log/output_flusher.ts

@@ -0,0 +1,48 @@
+import { Transform } from 'stream'
+
+const flusherSymbol = Symbol.for('@netlify/output-gate')
+
+/**
+ * Utility class for conditionally rendering certain output only if additional
+ * data flows through. The constructor takes a "buffer" function that renders
+ * the optional data. When flushed, that function is called.
+ */
+export class OutputFlusher {
+  private buffer: () => void
+
+  flushed: boolean
+
+  constructor(bufferFn: () => void) {
+    this.flushed = false
+    this.buffer = bufferFn
+  }
+
+  flush() {
+    if (!this.flushed) {
+      this.buffer()
+      this.flushed = true
+    }
+  }
+}
+
+/**
+ * A `Transform` stream that takes an `OutputFlusher` instance and flushes it
+ * whenever data flows through, before piping the data to its destination.
+ */
+export class OutputFlusherTransform extends Transform {
+  [flusherSymbol]: OutputFlusher
+
+  constructor(flusher: OutputFlusher) {
+    super()
+
+    this[flusherSymbol] = flusher
+  }
+
+  _transform(chunk: any, _: string, callback: () => void) {
+    this[flusherSymbol].flush()
+
+    this.push(chunk)
+
+    callback()
+  }
+}

packages/build/src/log/stream.js

@@ -1,15 +1,18 @@
 import { stdout, stderr } from 'process'
 import { promisify } from 'util'
 
+import { logsAreBuffered } from './logger.js'
+import { OutputFlusherTransform } from './output_flusher.js'
+
 // TODO: replace with `timers/promises` after dropping Node < 15.0.0
 const pSetTimeout = promisify(setTimeout)
 
 // We try to use `stdio: inherit` because it keeps `stdout/stderr` as `TTY`,
 // which solves many problems. However we can only do it in build.command.
 // Plugins have several events, so need to be switch on and off instead.
-// In buffer mode (`logs` not `undefined`), `pipe` is necessary.
+// In buffer mode, `pipe` is necessary.
 export const getBuildCommandStdio = function (logs) {
-  if (logs !== undefined) {
+  if (logsAreBuffered(logs)) {
     return 'pipe'
   }
 
@@ -18,7 +21,7 @@ export const getBuildCommandStdio = function (logs) {
 
 // Add build command output
 export const handleBuildCommandOutput = function ({ stdout: commandStdout, stderr: commandStderr }, logs) {
-  if (logs === undefined) {
+  if (!logsAreBuffered(logs)) {
     return
   }
 
@@ -35,28 +38,35 @@ const pushBuildCommandOutput = function (output, logsArray) {
 }
 
 // Start plugin step output
-export const pipePluginOutput = function (childProcess, logs) {
-  if (logs === undefined) {
-    return streamOutput(childProcess)
+export const pipePluginOutput = function (childProcess, logs, outputFlusher) {
+  if (!logsAreBuffered(logs)) {
+    return streamOutput(childProcess, outputFlusher)
   }
 
-  return pushOutputToLogs(childProcess, logs)
+  return pushOutputToLogs(childProcess, logs, outputFlusher)
 }
 
 // Stop streaming/buffering plugin step output
 export const unpipePluginOutput = async function (childProcess, logs, listeners) {
   // Let `childProcess` `stdout` and `stderr` flush before stopping redirecting
   await pSetTimeout(0)
 
-  if (logs === undefined) {
+  if (!logsAreBuffered(logs)) {
     return unstreamOutput(childProcess)
   }
 
   unpushOutputToLogs(childProcess, logs, listeners)
 }
 
 // Usually, we stream stdout/stderr because it is more efficient
-const streamOutput = function (childProcess) {
+const streamOutput = function (childProcess, outputFlusher) {
+  if (outputFlusher) {
+    childProcess.stdout.pipe(new OutputFlusherTransform(outputFlusher)).pipe(stdout)
+    childProcess.stderr.pipe(new OutputFlusherTransform(outputFlusher)).pipe(stderr)
+
+    return
+  }
+
   childProcess.stdout.pipe(stdout)
   childProcess.stderr.pipe(stderr)
 }
@@ -67,15 +77,21 @@ const unstreamOutput = function (childProcess) {
 }
 
 // In tests, we push to the `logs` array instead
-const pushOutputToLogs = function (childProcess, logs) {
-  const stdoutListener = logsListener.bind(null, logs.stdout)
-  const stderrListener = logsListener.bind(null, logs.stderr)
+const pushOutputToLogs = function (childProcess, logs, outputFlusher) {
+  const stdoutListener = logsListener.bind(null, logs.stdout, outputFlusher)
+  const stderrListener = logsListener.bind(null, logs.stderr, outputFlusher)
+
   childProcess.stdout.on('data', stdoutListener)
   childProcess.stderr.on('data', stderrListener)
+
   return { stdoutListener, stderrListener }
 }
 
-const logsListener = function (logs, chunk) {
+const logsListener = function (logs, outputFlusher, chunk) {
+  if (outputFlusher) {
+    outputFlusher.flush()
+  }
+
   logs.push(chunk.toString().trimEnd())
 }
 

packages/build/src/plugins_core/deploy/index.js

@@ -28,6 +28,9 @@ const coreStep = async function ({
 }) {
   const client = createBuildbotClient(buildbotServerSocket)
   try {
+    // buildbot will emit logs. Flush the output to preserve the right order.
+    logs?.outputFlusher?.flush()
+
     await connectBuildbotClient(client)
     await saveUpdatedConfig({
       configMutations,

packages/build/src/plugins_core/pre_cleanup/index.ts

@@ -24,4 +24,5 @@ export const preCleanup: CoreStep = {
   coreStepName: 'Pre cleanup',
   coreStepDescription: () => 'Cleaning up leftover files from previous builds',
   condition: blobsPresent,
+  quiet: true,
 }

packages/build/src/plugins_core/pre_dev_cleanup/index.ts

@@ -67,4 +67,5 @@ export const preDevCleanup: CoreStep = {
   coreStepName: 'Pre Dev cleanup',
   coreStepDescription: () => 'Cleaning up leftover files from previous builds',
   condition,
+  quiet: true,
 }