✨ [filesystem] Support for tarfs

changes/20250110140409.feature

@@ -0,0 +1 @@
+:sparkles: `[filesystem]` Support for `tarfs`

utils/filesystem/filesystem.go

@@ -23,6 +23,7 @@ const (
 	Embed
 	Custom
 	ZipFS
+	TarFS
 )
 
 var (
@@ -62,6 +63,23 @@ func NewZipFileSystemFromStandardFileSystem(source string, limits ILimits) (IClo
 	return NewZipFileSystem(NewStandardFileSystem(), source, limits)
 }
 
+// NewTarFileSystem returns a filesystem over the contents of a .tar file.
+// Warning: After use of the filesystem, it is crucial to close the tar file (tarFile) which has been opened from source for the entirety of the filesystem use session.
+// fs corresponds to the filesystem to use to find the tar file.
+func NewTarFileSystem(fs FS, source string, limits ILimits) (squashFS ICloseableFS, tarFile File, err error) {
+	wrapped, tarFile, err := newTarFSAdapterFromFilePath(fs, source, limits)
+	if err != nil {
+		return
+	}
+	squashFS = NewCloseableVirtualFileSystem(wrapped, TarFS, tarFile, fmt.Sprintf(".tar file `%v`", source), IdentityPathConverterFunc)
+	return
+}
+
+// NewTarFileSystemFromStandardFileSystem returns a tar filesystem similar to NewTarFileSystem but assumes the tar file described by source can be found on the standard file system.
+func NewTarFileSystemFromStandardFileSystem(source string, limits ILimits) (ICloseableFS, File, error) {
+	return NewTarFileSystem(NewStandardFileSystem(), source, limits)
+}
+
 func NewFs(fsType FilesystemType) FS {
 	switch fsType {
 	case StandardFS:

utils/filesystem/tar.go

@@ -0,0 +1,48 @@
+package filesystem
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+)
+
+func newTarReader(fs FS, source string, limits ILimits, currentDepth int64) (tarReader *tar.Reader, file File, err error) {
+	if fs == nil {
+		err = fmt.Errorf("%w: missing file system", commonerrors.ErrUndefined)
+		return
+	}
+	if limits == nil {
+		err = fmt.Errorf("%w: missing file system limits", commonerrors.ErrUndefined)
+		return
+	}
+	if limits.Apply() && limits.GetMaxDepth() >= 0 && currentDepth > limits.GetMaxDepth() {
+		err = fmt.Errorf("%w: depth [%v] of tar file [%v] is beyond allowed limits (max: %v)", commonerrors.ErrTooLarge, currentDepth, source, limits.GetMaxDepth())
+		return
+	}
+
+	if !fs.Exists(source) {
+		err = fmt.Errorf("%w: could not find archive [%v]", commonerrors.ErrNotFound, source)
+		return
+	}
+
+	info, err := fs.Lstat(source)
+	if err != nil {
+		return
+	}
+	file, err = fs.GenericOpen(source)
+	if err != nil {
+		return
+	}
+
+	tarFileSize := info.Size()
+
+	if limits.Apply() && tarFileSize > limits.GetMaxFileSize() {
+		err = fmt.Errorf("%w: tar file [%v] is too big (%v B) and beyond limits (max: %v B)", commonerrors.ErrTooLarge, source, tarFileSize, limits.GetMaxFileSize())
+		return
+	}
+
+	tarReader = tar.NewReader(file)
+
+	return
+}

utils/filesystem/tarfs.go

@@ -0,0 +1,35 @@
+package filesystem
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/afero/tarfs"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+)
+
+func newTarFSAdapterFromReader(reader *tar.Reader) (afero.Fs, error) {
+	if reader == nil {
+		return nil, fmt.Errorf("%w: missing reader", commonerrors.ErrUndefined)
+	}
+	return afero.NewReadOnlyFs(tarfs.New(reader)), nil
+}
+
+func newTarFSAdapterFromFilePath(fs FS, tarFilePath string, limits ILimits) (tarFs afero.Fs, tarFile File, err error) {
+	if fs == nil {
+		err = fmt.Errorf("%w: missing filesystem to use for finding the tar file", commonerrors.ErrUndefined)
+		return
+	}
+	tarReader, tarFile, err := newTarReader(fs, tarFilePath, limits, 0)
+	if err != nil && tarFile != nil {
+		subErr := tarFile.Close()
+		if subErr == nil {
+			tarFile = nil
+		}
+		return
+	}
+	tarFs, err = newTarFSAdapterFromReader(tarReader)
+	return
+}

utils/filesystem/tarfs_test.go

@@ -0,0 +1,235 @@
+package filesystem
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/go-faker/faker/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
+)
+
+const tarTestFileContent = "Test names:\r\nGeorge\r\nGeoffrey\r\nGonzo"
+
+var (
+	aferoTarTestContentTree = []string{
+		string(globalFileSystem.PathSeparator()),
+		filepath.Join("/", "sub"),
+		filepath.Join("/", "sub", "testDir2"),
+		filepath.Join("/", "sub", "testDir2", "testFile"),
+		filepath.Join("/", "testDir1"),
+		filepath.Join("/", "testDir1", "testFile"),
+		filepath.Join("/", "testFile"),
+	}
+)
+
+func Test_tarFs_Close(t *testing.T) {
+	fs, zipFile, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() {
+		if zipFile != nil {
+			_ = zipFile.Close()
+		}
+	}()
+	require.NotNil(t, zipFile)
+	_, err = fs.Stat("testuntar/test.txt")
+	assert.NoError(t, err)
+	require.NoError(t, fs.Close())
+	_, err = fs.Stat("testuntar/test.txt")
+	errortest.AssertErrorDescription(t, err, "closed")
+	require.NoError(t, fs.Close())
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Exists(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	assert.False(t, fs.Exists(faker.DomainName()))
+	// FIXME: enable when issue in afero is fixed https://github.com/spf13/afero/issues/395
+	// assert.True(t, fs.Exists(string(filepath.Separator)))
+	// assert.True(t, fs.Exists("/"))
+	assert.True(t, fs.Exists("testuntar/test.txt"))
+	assert.True(t, fs.Exists("testuntar/child.zip"))
+	assert.True(t, fs.Exists("testuntar/ไป ไหน มา.txt"))
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Exists_usingAferoTestTar(t *testing.T) {
+	// using afero test zip file
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	assert.False(t, fs.Exists(faker.DomainName()))
+	assert.True(t, fs.Exists(string(filepath.Separator)))
+	assert.True(t, fs.Exists("/"))
+
+	assert.True(t, fs.Exists("testDir1"))
+	assert.True(t, fs.Exists("testFile"))
+	assert.True(t, fs.Exists("testDir1/testFile"))
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_FileInfo(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	zfile, err := fs.Stat("/")
+	require.NoError(t, err)
+	assert.Equal(t, string(filepath.Separator), zfile.Name())
+	assert.True(t, zfile.IsDir())
+	assert.Zero(t, zfile.Size())
+
+	zfile, err = fs.Stat("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "test.txt", zfile.Name())
+	assert.False(t, zfile.IsDir())
+	assert.NotZero(t, zfile.Size())
+
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Browsing(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	empty, err := fs.IsEmpty(faker.DomainName())
+	require.NoError(t, err)
+	assert.True(t, empty)
+	empty, err = fs.IsEmpty("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.False(t, empty)
+	empty, err = fs.IsEmpty("testuntar/child.zip")
+	require.NoError(t, err)
+	assert.False(t, empty)
+	empty, err = fs.IsEmpty("testuntar/ไป ไหน มา.txt")
+	require.NoError(t, err)
+	assert.True(t, empty)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Browsing_usingAferoTestTar(t *testing.T) {
+	tarFs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = tarFs.Close() }()
+
+	// Warning: this assumes the walk function is executed in the same goroutine and not concurrently.
+	// If not, this list should be created with some thread access protection in place.
+	pathList := []string{}
+
+	var wFunc = func(path string, info fs.FileInfo, err error) error {
+		pathList = append(pathList, path)
+		return nil
+	}
+
+	require.NoError(t, tarFs.Walk("/", wFunc))
+	require.NoError(t, tarFs.Close())
+
+	sort.Strings(pathList)
+	sort.Strings(aferoTarTestContentTree)
+	assert.Equal(t, aferoTarTestContentTree, pathList)
+}
+
+func Test_tarFs_LS(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	files, err := fs.Ls("/")
+	require.NoError(t, err)
+	assert.NotZero(t, files)
+	assert.Contains(t, files, "testFile")
+
+	files, err = fs.Ls("sub/")
+	require.NoError(t, err)
+	assert.NotZero(t, files)
+	assert.Contains(t, files, "testDir2")
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_itemType(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	isFile, err := fs.IsFile("unzip")
+	require.NoError(t, err)
+	assert.False(t, isFile)
+	// FIXME: Enable when issue in afero is fixed https://github.com/spf13/afero/issues/395
+	// isDir, err := fs.IsDir("unzip")
+	// require.NoError(t, err)
+	// assert.True(t, isDir)
+	isFile, err = fs.IsFile("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.True(t, isFile)
+	isDir, err := fs.IsDir("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.False(t, isDir)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_itemType_usingAferoTestTar(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	isFile, err := fs.IsFile("testDir1")
+	require.NoError(t, err)
+	assert.False(t, isFile)
+	isDir, err := fs.IsDir("testDir1")
+	require.NoError(t, err)
+	assert.True(t, isDir)
+	isFile, err = fs.IsFile("testDir1/testFile")
+	require.NoError(t, err)
+	assert.True(t, isFile)
+	isDir, err = fs.IsDir("testDir1/testFile")
+	require.NoError(t, err)
+	assert.False(t, isDir)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Read(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	c, err := fs.ReadFile("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.Equal(t, tarTestFileContent, string(c))
+
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_not_supported(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	_, err = fs.TempDir("testdata", "aaaa")
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	f, err := fs.OpenFile("testuntar/test.txt", os.O_RDWR, os.FileMode(0600))
+	defer func() {
+		if f != nil {
+			_ = f.Close()
+		}
+	}()
+	require.Error(t, err)
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	err = fs.Chmod("testuntar/test.txt", os.FileMode(0600))
+	require.Error(t, err)
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	require.NoError(t, fs.Close())
+
+}