From c6cd557f8e869c88f73c51df9f6b1c0ba93d6340 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Sun, 6 Aug 2023 14:16:03 -0400 Subject: [PATCH] fix #3084, fix #3293: output files now have `hash` --- CHANGELOG.md | 8 ++++++ cmd/esbuild/service.go | 1 + lib/shared/common.ts | 3 +- lib/shared/stdio_protocol.ts | 1 + lib/shared/types.ts | 2 +- pkg/api/api.go | 1 + pkg/api/api_impl.go | 54 +++++++++++++++++------------------- pkg/api/serve_other.go | 16 +++++------ scripts/js-api-tests.js | 3 ++ 9 files changed, 50 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c3d9f2441..f0331e89a6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ This release fixes an issue that could cause esbuild to sometimes emit incorrect build output in cases where a file under the effect of `tsconfig.json` is inconsistently referenced through a symlink. It can happen when using `npm link` to create a symlink within `node_modules` to an unpublished package. The build result was non-deterministic because esbuild runs module resolution in parallel and the result of the `tsconfig.json` lookup depended on whether the import through the symlink or not through the symlink was resolved first. This problem was fixed by moving the `realpath` operation before the `tsconfig.json` lookup. +* Add a `hash` property to output files ([#3084](https://github.com/evanw/esbuild/issues/3084), [#3293](https://github.com/evanw/esbuild/issues/3293)) + + As a convenience, every output file in esbuild's API now includes a `hash` property that is a hash of the `contents` field. This is the hash that's used internally by esbuild to detect changes between builds for esbuild's live-reload feature. You may also use it to detect changes between your own builds if its properties are sufficient for your use case. + + This feature has been added directly to output file objects since it's just a hash of the `contents` field, so it makes conceptual sense to store it in the same location. Another benefit of putting it there instead of including it as a part of the watch mode API is that it can be used without watch mode enabled. You can use it to compare the output of two independent builds that were done at different times. + + The hash algorithm (currently [XXH64](https://xxhash.com/)) is implementation-dependent and may be changed at any time in between esbuild versions. If you don't like esbuild's choice of hash algorithm then you are welcome to hash the contents yourself instead. As with any hash algorithm, note that while two different hashes mean that the contents are different, two equal hashes do not necessarily mean that the contents are equal. You may still want to compare the contents in addition to the hashes to detect with certainty when output files have been changed. + ## 0.18.18 * Fix asset references with the `--line-limit` flag ([#3286](https://github.com/evanw/esbuild/issues/3286)) diff --git a/cmd/esbuild/service.go b/cmd/esbuild/service.go index babfaaaefed..b63f598d072 100644 --- a/cmd/esbuild/service.go +++ b/cmd/esbuild/service.go @@ -1260,6 +1260,7 @@ func encodeOutputFiles(outputFiles []api.OutputFile) []interface{} { values[i] = value value["path"] = outputFile.Path value["contents"] = outputFile.Contents + value["hash"] = outputFile.Hash } return values } diff --git a/lib/shared/common.ts b/lib/shared/common.ts index 6dc38e037ec..5c200b04b16 100644 --- a/lib/shared/common.ts +++ b/lib/shared/common.ts @@ -1750,13 +1750,14 @@ function sanitizeStringArray(values: any[], property: string): string[] { return result } -function convertOutputFiles({ path, contents }: protocol.BuildOutputFile): types.OutputFile { +function convertOutputFiles({ path, contents, hash }: protocol.BuildOutputFile): types.OutputFile { // The text is lazily-generated for performance reasons. If no one asks for // it, then it never needs to be generated. let text: string | null = null return { path, contents, + hash, get text() { // People want to be able to set "contents" and have esbuild automatically // derive "text" for them, so grab the contents off of this object instead diff --git a/lib/shared/stdio_protocol.ts b/lib/shared/stdio_protocol.ts index 6763ea35053..47acd923969 100644 --- a/lib/shared/stdio_protocol.ts +++ b/lib/shared/stdio_protocol.ts @@ -67,6 +67,7 @@ export interface OnEndResponse { export interface BuildOutputFile { path: string contents: Uint8Array + hash: string } export interface PingRequest { diff --git a/lib/shared/types.ts b/lib/shared/types.ts index 45a5166028e..48c99c1f6a5 100644 --- a/lib/shared/types.ts +++ b/lib/shared/types.ts @@ -209,8 +209,8 @@ export interface Location { export interface OutputFile { path: string - /** "text" as bytes */ contents: Uint8Array + hash: string /** "contents" as text (changes automatically with "contents") */ readonly text: string } diff --git a/pkg/api/api.go b/pkg/api/api.go index 70857ae07a5..e98eca178b8 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -369,6 +369,7 @@ type BuildResult struct { type OutputFile struct { Path string Contents []byte + Hash string } // Documentation: https://esbuild.github.io/api/#build diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go index 151d2642f42..a45d3fcca6f 100644 --- a/pkg/api/api_impl.go +++ b/pkg/api/api_impl.go @@ -5,6 +5,8 @@ package api import ( "bytes" + "encoding/base64" + "encoding/binary" "errors" "fmt" "io/ioutil" @@ -985,12 +987,16 @@ type internalContext struct { args rebuildArgs activeBuild *buildInProgress recentBuild *BuildResult - latestSummary buildSummary realFS fs.FS absWorkingDir string watcher *watcher handler *apiHandler didDispose bool + + // This saves just enough information to be able to compute a useful diff + // between two sets of output files. That way we don't need to hold both + // sets of output files in memory at once to compute a diff. + latestHashes map[string]string } func (ctx *internalContext) rebuild() rebuildState { @@ -1016,14 +1022,15 @@ func (ctx *internalContext) rebuild() rebuildState { args := ctx.args watcher := ctx.watcher handler := ctx.handler - oldSummary := ctx.latestSummary + oldHashes := ctx.latestHashes args.options.CancelFlag = &build.cancel ctx.mutex.Unlock() // Do the build without holding the mutex - build.state = rebuildImpl(args, oldSummary) + var newHashes map[string]string + build.state, newHashes = rebuildImpl(args, oldHashes) if handler != nil { - handler.broadcastBuildResult(build.state.result, build.state.summary) + handler.broadcastBuildResult(build.state.result, newHashes) } if watcher != nil { watcher.setWatchData(build.state.watchData) @@ -1034,7 +1041,7 @@ func (ctx *internalContext) rebuild() rebuildState { ctx.mutex.Lock() ctx.activeBuild = nil ctx.recentBuild = recentBuild - ctx.latestSummary = build.state.summary + ctx.latestHashes = newHashes ctx.mutex.Unlock() // Clear the recent build after it goes stale @@ -1459,12 +1466,11 @@ type rebuildArgs struct { type rebuildState struct { result BuildResult - summary buildSummary watchData fs.WatchData options config.Options } -func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { +func rebuildImpl(args rebuildArgs, oldHashes map[string]string) (rebuildState, map[string]string) { log := logger.NewStderrLog(args.logOptions) // All validation warnings are repeated for every rebuild @@ -1497,7 +1503,7 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { // The new build summary remains the same as the old one when there are // errors. A failed build shouldn't erase the previous successful build. - newSummary := oldSummary + newHashes := oldHashes // Stop now if there were errors if !log.HasErrors() { @@ -1515,17 +1521,23 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { result.Metafile = metafile // Populate the results to return + var hashBytes [8]byte result.OutputFiles = make([]OutputFile, len(results)) + newHashes = make(map[string]string) for i, item := range results { if args.options.WriteToStdout { item.AbsPath = "" } + hasher := xxhash.New() + hasher.Write(item.Contents) + hash := base64.RawStdEncoding.EncodeToString(binary.LittleEndian.AppendUint64(hashBytes[:0], hasher.Sum64())) result.OutputFiles[i] = OutputFile{ Path: item.AbsPath, Contents: item.Contents, + Hash: hash, } + newHashes[item.AbsPath] = hash } - newSummary = summarizeOutputFiles(result.OutputFiles) // Write output files before "OnEnd" callbacks run so they can expect // output files to exist on the file system. "OnEnd" callbacks can be @@ -1544,8 +1556,8 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { } else { // Delete old files that are no longer relevant var toDelete []string - for absPath := range oldSummary { - if _, ok := newSummary[absPath]; !ok { + for absPath := range oldHashes { + if _, ok := newHashes[absPath]; !ok { toDelete = append(toDelete, absPath) } } @@ -1558,7 +1570,7 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { defer waitGroup.Done() fs.BeforeFileOpen() defer fs.AfterFileClose() - if oldHash, ok := oldSummary[result.AbsPath]; ok && oldHash == newSummary[result.AbsPath] { + if oldHash, ok := oldHashes[result.AbsPath]; ok && oldHash == newHashes[result.AbsPath] { if contents, err := ioutil.ReadFile(result.AbsPath); err == nil && bytes.Equal(contents, result.Contents) { // Skip writing out files that haven't changed since last time return @@ -1665,10 +1677,9 @@ func rebuildImpl(args rebuildArgs, oldSummary buildSummary) rebuildState { return rebuildState{ result: result, - summary: newSummary, options: args.options, watchData: watchData, - } + }, newHashes } //////////////////////////////////////////////////////////////////////////////// @@ -2469,21 +2480,6 @@ func analyzeMetafileImpl(metafile string, opts AnalyzeMetafileOptions) string { return "" } -type buildSummary map[string]uint64 - -// This saves just enough information to be able to compute a useful diff -// between two sets of output files. That way we don't need to hold both -// sets of output files in memory at once to compute a diff. -func summarizeOutputFiles(outputFiles []OutputFile) buildSummary { - summary := make(map[string]uint64) - for _, outputFile := range outputFiles { - hash := xxhash.New() - hash.Write(outputFile.Contents) - summary[outputFile.Path] = hash.Sum64() - } - return summary -} - func stripDirPrefix(path string, prefix string, allowedSlashes string) (string, bool) { if strings.HasPrefix(path, prefix) { pathLen := len(path) diff --git a/pkg/api/serve_other.go b/pkg/api/serve_other.go index 3edcda2be3b..6e789621db6 100644 --- a/pkg/api/serve_other.go +++ b/pkg/api/serve_other.go @@ -50,7 +50,7 @@ type apiHandler struct { fallback string serveWaitGroup sync.WaitGroup activeStreams []chan serverSentEvent - buildSummary buildSummary + currentHashes map[string]string mutex sync.Mutex } @@ -405,7 +405,7 @@ func (h *apiHandler) serveEventStream(start time.Time, req *http.Request, res ht res.Write([]byte("500 - Event stream error")) } -func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSummary) { +func (h *apiHandler) broadcastBuildResult(result BuildResult, newHashes map[string]string) { h.mutex.Lock() var added []string @@ -429,11 +429,11 @@ func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSu // Diff the old and new states, but only if the build succeeded. We shouldn't // make it appear as if all files were removed when there is a build error. if len(result.Errors) == 0 { - oldSummary := h.buildSummary - h.buildSummary = newSummary + oldHashes := h.currentHashes + h.currentHashes = newHashes - for absPath, newHash := range newSummary { - if oldHash, ok := oldSummary[absPath]; !ok { + for absPath, newHash := range newHashes { + if oldHash, ok := oldHashes[absPath]; !ok { if url, ok := urlForPath(absPath); ok { added = append(added, url) } @@ -444,8 +444,8 @@ func (h *apiHandler) broadcastBuildResult(result BuildResult, newSummary buildSu } } - for absPath := range oldSummary { - if _, ok := newSummary[absPath]; !ok { + for absPath := range oldHashes { + if _, ok := newHashes[absPath]; !ok { if url, ok := urlForPath(absPath); ok { removed = append(removed, url) } diff --git a/scripts/js-api-tests.js b/scripts/js-api-tests.js index 88595423ee8..159b68817de 100644 --- a/scripts/js-api-tests.js +++ b/scripts/js-api-tests.js @@ -1638,8 +1638,10 @@ body { assert.strictEqual(value.outputFiles.length, 2) assert.strictEqual(value.outputFiles[0].path, output + '.map') assert.strictEqual(value.outputFiles[0].contents.constructor, Uint8Array) + assert.strictEqual(value.outputFiles[0].hash, 'BIjVBRZOQ5s') assert.strictEqual(value.outputFiles[1].path, output) assert.strictEqual(value.outputFiles[1].contents.constructor, Uint8Array) + assert.strictEqual(value.outputFiles[1].hash, 'zvyzJPvi96o') const sourceMap = JSON.parse(Buffer.from(value.outputFiles[0].contents).toString()) const js = Buffer.from(value.outputFiles[1].contents).toString() @@ -7061,6 +7063,7 @@ let syncTests = { assert.strictEqual(result.outputFiles[0].path, output) assert.strictEqual(result.outputFiles[0].text, text) assert.deepStrictEqual(result.outputFiles[0].contents, new Uint8Array(Buffer.from(text))) + assert.strictEqual(result.outputFiles[0].hash, 'H4KMzZ07fA0') }, async transformSyncJSMap({ esbuild }) {