From a481005d8fd6b5504a9ed139e576d5b6534959bb Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Fri, 4 Feb 2022 08:14:25 +0000 Subject: [PATCH] add zip file support --- internal/fs/fs_real.go | 13 +- internal/fs/fs_zip.go | 292 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 internal/fs/fs_zip.go diff --git a/internal/fs/fs_real.go b/internal/fs/fs_real.go index d0f294b1652..7a4cf4e8a42 100644 --- a/internal/fs/fs_real.go +++ b/internal/fs/fs_real.go @@ -110,12 +110,21 @@ func RealFS(options RealFSOptions) (FS, error) { watchData = make(map[string]privateWatchData) } - return &realFS{ + var result FS = &realFS{ entries: make(map[string]entriesOrErr), fp: fp, watchData: watchData, doNotCacheEntries: options.DoNotCache, - }, nil + } + + // Add a wrapper that lets us traverse into ".zip" files. This is what yarn + // uses as a package format when in yarn is in its "PnP" mode. + result = &zipFS{ + inner: result, + zipFiles: make(map[string]*zipFile), + } + + return result, nil } func (fs *realFS) ReadDirectory(dir string) (entries DirEntries, canonicalError error, originalError error) { diff --git a/internal/fs/fs_zip.go b/internal/fs/fs_zip.go new file mode 100644 index 00000000000..ad3f84958a7 --- /dev/null +++ b/internal/fs/fs_zip.go @@ -0,0 +1,292 @@ +package fs + +import ( + "archive/zip" + "io/ioutil" + "strings" + "sync" + "syscall" +) + +type zipFS struct { + inner FS + + zipFilesMutex sync.Mutex + zipFiles map[string]*zipFile +} + +type zipFile struct { + reader *zip.ReadCloser + err error + + dirs map[string]*compressedDir + files map[string]*compressedFile + wait sync.WaitGroup +} + +type compressedDir struct { + entries map[string]EntryKind + path string + + // Compatible entries are decoded lazily + mutex sync.Mutex + dirEntries DirEntries +} + +type compressedFile struct { + compressed *zip.File + + // The file is decompressed lazily + mutex sync.Mutex + contents string + err error + wasRead bool +} + +func (fs *zipFS) checkForZip(path string, kind EntryKind) (*zipFile, string) { + var zipPath string + var pathTail string + + // Do a quick check for a ".zip" in the path at all + path = strings.ReplaceAll(path, "\\", "/") + if i := strings.Index(path, ".zip/"); i != -1 { + zipPath = path[:i+len(".zip")] + pathTail = path[i+len(".zip/"):] + } else if kind == DirEntry && strings.HasSuffix(path, ".zip") { + zipPath = path + } else { + return nil, "" + } + + // If there is one, then check whether it's a file on the file system or not + fs.zipFilesMutex.Lock() + archive := fs.zipFiles[zipPath] + if archive != nil { + fs.zipFilesMutex.Unlock() + archive.wait.Wait() + } else { + archive = &zipFile{} + archive.wait.Add(1) + fs.zipFiles[zipPath] = archive + fs.zipFilesMutex.Unlock() + defer archive.wait.Done() + + // Try reading the zip archive if it's not in the cache + tryToReadZipArchive(zipPath, archive) + } + + if archive.err != nil { + return nil, "" + } + return archive, pathTail +} + +func tryToReadZipArchive(zipPath string, archive *zipFile) { + reader, err := zip.OpenReader(zipPath) + if err != nil { + archive.err = err + return + } + + dirs := make(map[string]*compressedDir) + files := make(map[string]*compressedFile) + + // Build an index of all files in the archive + for _, file := range reader.File { + baseName := file.Name + if strings.HasSuffix(baseName, "/") { + baseName = baseName[:len(baseName)-1] + } + dirPath := "" + if slash := strings.LastIndexByte(baseName, '/'); slash != -1 { + dirPath = baseName[:slash] + baseName = baseName[slash+1:] + } + if file.FileInfo().IsDir() { + // Handle a directory + lowerDir := strings.ToLower(dirPath) + if _, ok := dirs[lowerDir]; !ok { + dirs[lowerDir] = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + } + } else { + // Handle a file + files[strings.ToLower(file.Name)] = &compressedFile{compressed: file} + lowerDir := strings.ToLower(dirPath) + dir, ok := dirs[lowerDir] + if !ok { + dir = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + dirs[lowerDir] = dir + } + dir.entries[baseName] = FileEntry + } + } + + // Populate child directories + seeds := make([]string, 0, len(dirs)) + for dir := range dirs { + seeds = append(seeds, dir) + } + for _, baseName := range seeds { + for baseName != "" { + dirPath := "" + if slash := strings.LastIndexByte(baseName, '/'); slash != -1 { + dirPath = baseName[:slash] + baseName = baseName[slash+1:] + } + lowerDir := strings.ToLower(dirPath) + dir, ok := dirs[lowerDir] + if !ok { + dir = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + dirs[lowerDir] = dir + } + dir.entries[baseName] = DirEntry + baseName = dirPath + } + } + + archive.dirs = dirs + archive.files = files + archive.reader = reader +} + +func (fs *zipFS) ReadDirectory(path string) (entries DirEntries, canonicalError error, originalError error) { + entries, canonicalError, originalError = fs.inner.ReadDirectory(path) + if canonicalError != syscall.ENOENT && canonicalError != syscall.ENOTDIR { + return + } + + // If the directory doesn't exist, try reading from an enclosing zip archive + zip, pathTail := fs.checkForZip(path, DirEntry) + if zip == nil { + return + } + + // Does the zip archive have this directory? + dir, ok := zip.dirs[strings.ToLower(pathTail)] + if !ok { + return DirEntries{}, syscall.ENOENT, syscall.ENOENT + } + + // Check whether it has already been converted + dir.mutex.Lock() + defer dir.mutex.Unlock() + if dir.dirEntries.data != nil { + return dir.dirEntries, nil, nil + } + + // Otherwise, fill in the entries + dir.dirEntries = DirEntries{dir: path, data: make(map[string]*Entry, len(dir.entries))} + for name, kind := range dir.entries { + dir.dirEntries.data[strings.ToLower(name)] = &Entry{ + dir: path, + base: name, + kind: kind, + } + } + + return dir.dirEntries, nil, nil +} + +func (fs *zipFS) ReadFile(path string) (contents string, canonicalError error, originalError error) { + contents, canonicalError, originalError = fs.inner.ReadFile(path) + if canonicalError != syscall.ENOENT { + return + } + + // If the file doesn't exist, try reading from an enclosing zip archive + zip, pathTail := fs.checkForZip(path, FileEntry) + if zip == nil { + return + } + + // Does the zip archive have this file? + file, ok := zip.files[strings.ToLower(pathTail)] + if !ok { + return "", syscall.ENOENT, syscall.ENOENT + } + + // Check whether it has already been read + file.mutex.Lock() + defer file.mutex.Unlock() + if file.wasRead { + return file.contents, file.err, file.err + } + file.wasRead = true + + // If not, try to open it + reader, err := file.compressed.Open() + if err != nil { + file.err = err + return "", err, err + } + defer reader.Close() + + // Then try to read it + bytes, err := ioutil.ReadAll(reader) + if err != nil { + file.err = err + return "", err, err + } + + file.contents = string(bytes) + return file.contents, nil, nil +} + +func (fs *zipFS) OpenFile(path string) (result OpenedFile, canonicalError error, originalError error) { + result, canonicalError, originalError = fs.inner.OpenFile(path) + return +} + +func (fs *zipFS) ModKey(path string) (modKey ModKey, err error) { + modKey, err = fs.inner.ModKey(path) + return +} + +func (fs *zipFS) IsAbs(path string) bool { + return fs.inner.IsAbs(path) +} + +func (fs *zipFS) Abs(path string) (string, bool) { + return fs.inner.Abs(path) +} + +func (fs *zipFS) Dir(path string) string { + return fs.inner.Dir(path) +} + +func (fs *zipFS) Base(path string) string { + return fs.inner.Base(path) +} + +func (fs *zipFS) Ext(path string) string { + return fs.inner.Ext(path) +} + +func (fs *zipFS) Join(parts ...string) string { + return fs.inner.Join(parts...) +} + +func (fs *zipFS) Cwd() string { + return fs.inner.Cwd() +} + +func (fs *zipFS) Rel(base string, target string) (string, bool) { + return fs.inner.Rel(base, target) +} + +func (fs *zipFS) kind(dir string, base string) (symlink string, kind EntryKind) { + return fs.inner.kind(dir, base) +} + +func (fs *zipFS) WatchData() WatchData { + return fs.inner.WatchData() +}