fix #3084, fix #3293: output files now have hash

CHANGELOG.md

@@ -6,6 +6,14 @@
 
     This release fixes an issue that could cause esbuild to sometimes emit incorrect build output in cases where a file under the effect of `tsconfig.json` is inconsistently referenced through a symlink. It can happen when using `npm link` to create a symlink within `node_modules` to an unpublished package. The build result was non-deterministic because esbuild runs module resolution in parallel and the result of the `tsconfig.json` lookup depended on whether the import through the symlink or not through the symlink was resolved first. This problem was fixed by moving the `realpath` operation before the `tsconfig.json` lookup.
 
+* Add a `hash` property to output files ([#3084](https://github.com/evanw/esbuild/issues/3084), [#3293](https://github.com/evanw/esbuild/issues/3293))
+
+    As a convenience, every output file in esbuild's API now includes a `hash` property that is a hash of the `contents` field. This is the hash that's used internally by esbuild to detect changes between builds for esbuild's live-reload feature. You may also use it to detect changes between your own builds if its properties are sufficient for your use case.
+
+    This feature has been added directly to output file objects since it's just a hash of the `contents` field, so it makes conceptual sense to store it in the same location. Another benefit of putting it there instead of including it as a part of the watch mode API is that it can be used without watch mode enabled. You can use it to compare the output of two independent builds that were done at different times.
+
+    The hash algorithm (currently [XXH64](https://xxhash.com/)) is implementation-dependent and may be changed at any time in between esbuild versions. If you don't like esbuild's choice of hash algorithm then you are welcome to hash the contents yourself instead. As with any hash algorithm, note that while two different hashes mean that the contents are different, two equal hashes do not necessarily mean that the contents are equal. You may still want to compare the contents in addition to the hashes to detect with certainty when output files have been changed.
+
 ## 0.18.18
 
 * Fix asset references with the `--line-limit` flag ([#3286](https://github.com/evanw/esbuild/issues/3286))

cmd/esbuild/service.go

@@ -1260,6 +1260,7 @@ func encodeOutputFiles(outputFiles []api.OutputFile) []interface{} {
 		values[i] = value
 		value["path"] = outputFile.Path
 		value["contents"] = outputFile.Contents
+		value["hash"] = outputFile.Hash
 	}
 	return values
 }

lib/shared/common.ts

@@ -1750,13 +1750,14 @@ function sanitizeStringArray(values: any[], property: string): string[] {
   return result
 }
 
-function convertOutputFiles({ path, contents }: protocol.BuildOutputFile): types.OutputFile {
+function convertOutputFiles({ path, contents, hash }: protocol.BuildOutputFile): types.OutputFile {
   // The text is lazily-generated for performance reasons. If no one asks for
   // it, then it never needs to be generated.
   let text: string | null = null
   return {
     path,
     contents,
+    hash,
     get text() {
       // People want to be able to set "contents" and have esbuild automatically
       // derive "text" for them, so grab the contents off of this object instead

lib/shared/stdio_protocol.ts

@@ -67,6 +67,7 @@ export interface OnEndResponse {
 export interface BuildOutputFile {
   path: string
   contents: Uint8Array
+  hash: string
 }
 
 export interface PingRequest {

lib/shared/types.ts

@@ -209,8 +209,8 @@ export interface Location {
 
 export interface OutputFile {
   path: string
-  /** "text" as bytes */
   contents: Uint8Array
+  hash: string
   /** "contents" as text (changes automatically with "contents") */
   readonly text: string
 }

pkg/api/api.go

@@ -369,6 +369,7 @@ type BuildResult struct {
 type OutputFile struct {
 	Path     string
 	Contents []byte
+	Hash     string
 }
 
 // Documentation: https://esbuild.github.io/api/#build

pkg/api/api_impl.go

@@ -5,6 +5,8 @@ package api
 
 import (
 	"bytes"
+	"encoding/base64"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -985,12 +987,16 @@ type internalContext struct {
 	args          rebuildArgs
 	activeBuild   *buildInProgress
 	recentBuild   *BuildResult
-	latestSummary buildSummary
 	realFS        fs.FS
 	absWorkingDir string
 	watcher       *watcher
 	handler       *apiHandler
 	didDispose    bool
+
+	// This saves just enough information to be able to compute a useful diff
+	// between two sets of output files. That way we don't need to hold both
+	// sets of output files in memory at once to compute a diff.
+	latestHashes map[string]string
 }
 
 func (ctx *internalContext) rebuild() rebuildState {
@@ -1016,14 +1022,15 @@ func (ctx *internalContext) rebuild() rebuildState {
 	args := ctx.args
 	watcher := ctx.watcher
 	handler := ctx.handler
-	oldSummary := ctx.latestSummary
+	oldHashes := ctx.latestHashes
 	args.options.CancelFlag = &build.cancel
 	ctx.mutex.Unlock()
 
 	// Do the build without holding the mutex
-	build.state = rebuildImpl(args, oldSummary)
+	var newHashes map[string]string
+	build.state, newHashes = rebuildImpl(args, oldHashes)
 	if handler != nil {
-		handler.broadcastBuildResult(build.state.result, build.state.summary)
+		handler.broadcastBuildResult(build.state.result, newHashes)
 	}
 	if watcher != nil {
 		watcher.setWatchData(build.state.watchData)
@@ -1034,7 +1041,7 @@ func (ctx *internalContext) rebuild() rebuildState {
 	ctx.mutex.Lock()
 	ctx.activeBuild = nil
 	ctx.recentBuild = recentBuild
-	ctx.latestSummary = build.state.summary
+	ctx.latestHashes = newHashes
 	ctx.mutex.Unlock()
 
 	// Clear the recent build after it goes stale
@@ -1459,12 +1466,11 @@ type rebuildArgs struct {
 
 type rebuildState struct {
 	result    BuildResult
-	summary   buildSummary
 	watchData fs.WatchData
 	options   config.Options
 }
 
-func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
+func rebuildImpl(args rebuildArgs, oldHashes map[string]string) (rebuildState, map[string]string) {
 	log := logger.NewStderrLog(args.logOptions)
 
 	// All validation warnings are repeated for every rebuild
@@ -1497,7 +1503,7 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
 
 	// The new build summary remains the same as the old one when there are
 	// errors. A failed build shouldn't erase the previous successful build.
-	newSummary := oldSummary
+	newHashes := oldHashes
 
 	// Stop now if there were errors
 	if !log.HasErrors() {
@@ -1515,17 +1521,23 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
 			result.Metafile = metafile
 
 			// Populate the results to return
+			var hashBytes [8]byte
 			result.OutputFiles = make([]OutputFile, len(results))
+			newHashes = make(map[string]string)
 			for i, item := range results {
 				if args.options.WriteToStdout {
 					item.AbsPath = "<stdout>"
 				}
+				hasher := xxhash.New()
+				hasher.Write(item.Contents)
+				hash := base64.RawStdEncoding.EncodeToString(binary.LittleEndian.AppendUint64(hashBytes[:0], hasher.Sum64()))
 				result.OutputFiles[i] = OutputFile{
 					Path:     item.AbsPath,
 					Contents: item.Contents,
+					Hash:     hash,
 				}
+				newHashes[item.AbsPath] = hash
 			}
-			newSummary = summarizeOutputFiles(result.OutputFiles)
 
 			// Write output files before "OnEnd" callbacks run so they can expect
 			// output files to exist on the file system. "OnEnd" callbacks can be
@@ -1544,8 +1556,8 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
 				} else {
 					// Delete old files that are no longer relevant
 					var toDelete []string
-					for absPath := range oldSummary {
-						if _, ok := newSummary[absPath]; !ok {
+					for absPath := range oldHashes {
+						if _, ok := newHashes[absPath]; !ok {
 							toDelete = append(toDelete, absPath)
 						}
 					}
@@ -1558,7 +1570,7 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
 							defer waitGroup.Done()
 							fs.BeforeFileOpen()
 							defer fs.AfterFileClose()
-							if oldHash, ok := oldSummary[result.AbsPath]; ok && oldHash == newSummary[result.AbsPath] {
+							if oldHash, ok := oldHashes[result.AbsPath]; ok && oldHash == newHashes[result.AbsPath] {
 								if contents, err := ioutil.ReadFile(result.AbsPath); err == nil && bytes.Equal(contents, result.Contents) {
 									// Skip writing out files that haven't changed since last time
 									return
@@ -1665,10 +1677,9 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState {
 
 	return rebuildState{
 		result:    result,
-		summary:   newSummary,
 		options:   args.options,
 		watchData: watchData,
-	}
+	}, newHashes
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -2469,21 +2480,6 @@ func analyzeMetafileImpl(metafile string, opts AnalyzeMetafileOptions) string {
 	return ""
 }
 
-type buildSummary map[string]uint64
-
-// This saves just enough information to be able to compute a useful diff
-// between two sets of output files. That way we don't need to hold both
-// sets of output files in memory at once to compute a diff.
-func summarizeOutputFiles(outputFiles []OutputFile) buildSummary {
-	summary := make(map[string]uint64)
-	for _, outputFile := range outputFiles {
-		hash := xxhash.New()
-		hash.Write(outputFile.Contents)
-		summary[outputFile.Path] = hash.Sum64()
-	}
-	return summary
-}
-
 func stripDirPrefix(path string, prefix string, allowedSlashes string) (string, bool) {
 	if strings.HasPrefix(path, prefix) {
 		pathLen := len(path)

pkg/api/serve_other.go

@@ -50,7 +50,7 @@ type apiHandler struct {
 	fallback         string
 	serveWaitGroup   sync.WaitGroup
 	activeStreams    []chan serverSentEvent
-	buildSummary     buildSummary
+	currentHashes    map[string]string
 	mutex            sync.Mutex
 }
 
@@ -405,7 +405,7 @@ func (h *apiHandler) serveEventStream(start time.Time, req *http.Request, res ht
 	res.Write([]byte("500 - Event stream error"))
 }
 
-func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSummary) {
+func (h *apiHandler) broadcastBuildResult(result BuildResult, newHashes map[string]string) {
 	h.mutex.Lock()
 
 	var added []string
@@ -429,11 +429,11 @@ func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSu
 	// Diff the old and new states, but only if the build succeeded. We shouldn't
 	// make it appear as if all files were removed when there is a build error.
 	if len(result.Errors) == 0 {
-		oldSummary := h.buildSummary
-		h.buildSummary = newSummary
+		oldHashes := h.currentHashes
+		h.currentHashes = newHashes
 
-		for absPath, newHash := range newSummary {
-			if oldHash, ok := oldSummary[absPath]; !ok {
+		for absPath, newHash := range newHashes {
+			if oldHash, ok := oldHashes[absPath]; !ok {
 				if url, ok := urlForPath(absPath); ok {
 					added = append(added, url)
 				}
@@ -444,8 +444,8 @@ func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSu
 			}
 		}
 
-		for absPath := range oldSummary {
-			if _, ok := newSummary[absPath]; !ok {
+		for absPath := range oldHashes {
+			if _, ok := newHashes[absPath]; !ok {
 				if url, ok := urlForPath(absPath); ok {
 					removed = append(removed, url)
 				}

scripts/js-api-tests.js

@@ -1638,8 +1638,10 @@ body {
     assert.strictEqual(value.outputFiles.length, 2)
     assert.strictEqual(value.outputFiles[0].path, output + '.map')
     assert.strictEqual(value.outputFiles[0].contents.constructor, Uint8Array)
+    assert.strictEqual(value.outputFiles[0].hash, 'BIjVBRZOQ5s')
     assert.strictEqual(value.outputFiles[1].path, output)
     assert.strictEqual(value.outputFiles[1].contents.constructor, Uint8Array)
+    assert.strictEqual(value.outputFiles[1].hash, 'zvyzJPvi96o')
 
     const sourceMap = JSON.parse(Buffer.from(value.outputFiles[0].contents).toString())
     const js = Buffer.from(value.outputFiles[1].contents).toString()
@@ -7061,6 +7063,7 @@ let syncTests = {
     assert.strictEqual(result.outputFiles[0].path, output)
     assert.strictEqual(result.outputFiles[0].text, text)
     assert.deepStrictEqual(result.outputFiles[0].contents, new Uint8Array(Buffer.from(text)))
+    assert.strictEqual(result.outputFiles[0].hash, 'H4KMzZ07fA0')
   },
 
   async transformSyncJSMap({ esbuild }) {