From 05daa8956a280deba07d3368f08f7b1a008f8787 Mon Sep 17 00:00:00 2001
From: Adrien CABARBAYE <adrien.cabarbaye@arm.com>
Date: Fri, 10 Jan 2025 14:04:57 +0000
Subject: [PATCH] :sparkles: `[filesystem]` Support for `tarfs`

---
 changes/20250110140409.feature          |   1 +
 utils/filesystem/filesystem.go          |  18 ++
 utils/filesystem/tar.go                 |  48 +++++
 utils/filesystem/tarfs.go               |  35 ++++
 utils/filesystem/tarfs_test.go          | 235 ++++++++++++++++++++++++
 utils/filesystem/testdata/t.tar         | Bin 0 -> 30720 bytes
 utils/filesystem/testdata/testuntar.tar | Bin 0 -> 6656 bytes
 7 files changed, 337 insertions(+)
 create mode 100644 changes/20250110140409.feature
 create mode 100644 utils/filesystem/tar.go
 create mode 100644 utils/filesystem/tarfs.go
 create mode 100644 utils/filesystem/tarfs_test.go
 create mode 100644 utils/filesystem/testdata/t.tar
 create mode 100644 utils/filesystem/testdata/testuntar.tar

diff --git a/changes/20250110140409.feature b/changes/20250110140409.feature
new file mode 100644
index 0000000000..243038d152
--- /dev/null
+++ b/changes/20250110140409.feature
@@ -0,0 +1 @@
+:sparkles: `[filesystem]` Support for `tarfs`
diff --git a/utils/filesystem/filesystem.go b/utils/filesystem/filesystem.go
index 7e393b2fdb..1380937329 100644
--- a/utils/filesystem/filesystem.go
+++ b/utils/filesystem/filesystem.go
@@ -23,6 +23,7 @@ const (
 	Embed
 	Custom
 	ZipFS
+	TarFS
 )
 
 var (
@@ -62,6 +63,23 @@ func NewZipFileSystemFromStandardFileSystem(source string, limits ILimits) (IClo
 	return NewZipFileSystem(NewStandardFileSystem(), source, limits)
 }
 
+// NewTarFileSystem returns a filesystem over the contents of a .tar file.
+// Warning: After use of the filesystem, it is crucial to close the tar file (tarFile) which has been opened from source for the entirety of the filesystem use session.
+// fs corresponds to the filesystem to use to find the tar file.
+func NewTarFileSystem(fs FS, source string, limits ILimits) (squashFS ICloseableFS, tarFile File, err error) {
+	wrapped, tarFile, err := newTarFSAdapterFromFilePath(fs, source, limits)
+	if err != nil {
+		return
+	}
+	squashFS = NewCloseableVirtualFileSystem(wrapped, TarFS, tarFile, fmt.Sprintf(".tar file `%v`", source), IdentityPathConverterFunc)
+	return
+}
+
+// NewTarFileSystemFromStandardFileSystem returns a tar filesystem similar to NewTarFileSystem but assumes the tar file described by source can be found on the standard file system.
+func NewTarFileSystemFromStandardFileSystem(source string, limits ILimits) (ICloseableFS, File, error) {
+	return NewTarFileSystem(NewStandardFileSystem(), source, limits)
+}
+
 func NewFs(fsType FilesystemType) FS {
 	switch fsType {
 	case StandardFS:
diff --git a/utils/filesystem/tar.go b/utils/filesystem/tar.go
new file mode 100644
index 0000000000..d82c20911f
--- /dev/null
+++ b/utils/filesystem/tar.go
@@ -0,0 +1,48 @@
+package filesystem
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+)
+
+func newTarReader(fs FS, source string, limits ILimits, currentDepth int64) (tarReader *tar.Reader, file File, err error) {
+	if fs == nil {
+		err = fmt.Errorf("%w: missing file system", commonerrors.ErrUndefined)
+		return
+	}
+	if limits == nil {
+		err = fmt.Errorf("%w: missing file system limits", commonerrors.ErrUndefined)
+		return
+	}
+	if limits.Apply() && limits.GetMaxDepth() >= 0 && currentDepth > limits.GetMaxDepth() {
+		err = fmt.Errorf("%w: depth [%v] of tar file [%v] is beyond allowed limits (max: %v)", commonerrors.ErrTooLarge, currentDepth, source, limits.GetMaxDepth())
+		return
+	}
+
+	if !fs.Exists(source) {
+		err = fmt.Errorf("%w: could not find archive [%v]", commonerrors.ErrNotFound, source)
+		return
+	}
+
+	info, err := fs.Lstat(source)
+	if err != nil {
+		return
+	}
+	file, err = fs.GenericOpen(source)
+	if err != nil {
+		return
+	}
+
+	tarFileSize := info.Size()
+
+	if limits.Apply() && tarFileSize > limits.GetMaxFileSize() {
+		err = fmt.Errorf("%w: tar file [%v] is too big (%v B) and beyond limits (max: %v B)", commonerrors.ErrTooLarge, source, tarFileSize, limits.GetMaxFileSize())
+		return
+	}
+
+	tarReader = tar.NewReader(file)
+
+	return
+}
diff --git a/utils/filesystem/tarfs.go b/utils/filesystem/tarfs.go
new file mode 100644
index 0000000000..ce95b081f4
--- /dev/null
+++ b/utils/filesystem/tarfs.go
@@ -0,0 +1,35 @@
+package filesystem
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/afero/tarfs"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+)
+
+func newTarFSAdapterFromReader(reader *tar.Reader) (afero.Fs, error) {
+	if reader == nil {
+		return nil, fmt.Errorf("%w: missing reader", commonerrors.ErrUndefined)
+	}
+	return afero.NewReadOnlyFs(tarfs.New(reader)), nil
+}
+
+func newTarFSAdapterFromFilePath(fs FS, tarFilePath string, limits ILimits) (tarFs afero.Fs, tarFile File, err error) {
+	if fs == nil {
+		err = fmt.Errorf("%w: missing filesystem to use for finding the tar file", commonerrors.ErrUndefined)
+		return
+	}
+	tarReader, tarFile, err := newTarReader(fs, tarFilePath, limits, 0)
+	if err != nil && tarFile != nil {
+		subErr := tarFile.Close()
+		if subErr == nil {
+			tarFile = nil
+		}
+		return
+	}
+	tarFs, err = newTarFSAdapterFromReader(tarReader)
+	return
+}
diff --git a/utils/filesystem/tarfs_test.go b/utils/filesystem/tarfs_test.go
new file mode 100644
index 0000000000..1f036d77b0
--- /dev/null
+++ b/utils/filesystem/tarfs_test.go
@@ -0,0 +1,235 @@
+package filesystem
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/go-faker/faker/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
+)
+
+const tarTestFileContent = "Test names:\r\nGeorge\r\nGeoffrey\r\nGonzo"
+
+var (
+	aferoTarTestContentTree = []string{
+		string(globalFileSystem.PathSeparator()),
+		filepath.Join("/", "sub"),
+		filepath.Join("/", "sub", "testDir2"),
+		filepath.Join("/", "sub", "testDir2", "testFile"),
+		filepath.Join("/", "testDir1"),
+		filepath.Join("/", "testDir1", "testFile"),
+		filepath.Join("/", "testFile"),
+	}
+)
+
+func Test_tarFs_Close(t *testing.T) {
+	fs, zipFile, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() {
+		if zipFile != nil {
+			_ = zipFile.Close()
+		}
+	}()
+	require.NotNil(t, zipFile)
+	_, err = fs.Stat("testuntar/test.txt")
+	assert.NoError(t, err)
+	require.NoError(t, fs.Close())
+	_, err = fs.Stat("testuntar/test.txt")
+	errortest.AssertErrorDescription(t, err, "closed")
+	require.NoError(t, fs.Close())
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Exists(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	assert.False(t, fs.Exists(faker.DomainName()))
+	// FIXME: enable when issue in afero is fixed https://github.com/spf13/afero/issues/395
+	// assert.True(t, fs.Exists(string(filepath.Separator)))
+	// assert.True(t, fs.Exists("/"))
+	assert.True(t, fs.Exists("testuntar/test.txt"))
+	assert.True(t, fs.Exists("testuntar/child.zip"))
+	assert.True(t, fs.Exists("testuntar/ไป ไหน มา.txt"))
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Exists_usingAferoTestTar(t *testing.T) {
+	// using afero test zip file
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	assert.False(t, fs.Exists(faker.DomainName()))
+	assert.True(t, fs.Exists(string(filepath.Separator)))
+	assert.True(t, fs.Exists("/"))
+
+	assert.True(t, fs.Exists("testDir1"))
+	assert.True(t, fs.Exists("testFile"))
+	assert.True(t, fs.Exists("testDir1/testFile"))
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_FileInfo(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	zfile, err := fs.Stat("/")
+	require.NoError(t, err)
+	assert.Equal(t, string(filepath.Separator), zfile.Name())
+	assert.True(t, zfile.IsDir())
+	assert.Zero(t, zfile.Size())
+
+	zfile, err = fs.Stat("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "test.txt", zfile.Name())
+	assert.False(t, zfile.IsDir())
+	assert.NotZero(t, zfile.Size())
+
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Browsing(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	empty, err := fs.IsEmpty(faker.DomainName())
+	require.NoError(t, err)
+	assert.True(t, empty)
+	empty, err = fs.IsEmpty("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.False(t, empty)
+	empty, err = fs.IsEmpty("testuntar/child.zip")
+	require.NoError(t, err)
+	assert.False(t, empty)
+	empty, err = fs.IsEmpty("testuntar/ไป ไหน มา.txt")
+	require.NoError(t, err)
+	assert.True(t, empty)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Browsing_usingAferoTestTar(t *testing.T) {
+	tarFs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = tarFs.Close() }()
+
+	// Warning: this assumes the walk function is executed in the same goroutine and not concurrently.
+	// If not, this list should be created with some thread access protection in place.
+	pathList := []string{}
+
+	var wFunc = func(path string, info fs.FileInfo, err error) error {
+		pathList = append(pathList, path)
+		return nil
+	}
+
+	require.NoError(t, tarFs.Walk("/", wFunc))
+	require.NoError(t, tarFs.Close())
+
+	sort.Strings(pathList)
+	sort.Strings(aferoTarTestContentTree)
+	assert.Equal(t, aferoTarTestContentTree, pathList)
+}
+
+func Test_tarFs_LS(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	files, err := fs.Ls("/")
+	require.NoError(t, err)
+	assert.NotZero(t, files)
+	assert.Contains(t, files, "testFile")
+
+	files, err = fs.Ls("sub/")
+	require.NoError(t, err)
+	assert.NotZero(t, files)
+	assert.Contains(t, files, "testDir2")
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_itemType(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	isFile, err := fs.IsFile("unzip")
+	require.NoError(t, err)
+	assert.False(t, isFile)
+	// FIXME: Enable when issue in afero is fixed https://github.com/spf13/afero/issues/395
+	// isDir, err := fs.IsDir("unzip")
+	// require.NoError(t, err)
+	// assert.True(t, isDir)
+	isFile, err = fs.IsFile("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.True(t, isFile)
+	isDir, err := fs.IsDir("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.False(t, isDir)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_itemType_usingAferoTestTar(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "t.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	isFile, err := fs.IsFile("testDir1")
+	require.NoError(t, err)
+	assert.False(t, isFile)
+	isDir, err := fs.IsDir("testDir1")
+	require.NoError(t, err)
+	assert.True(t, isDir)
+	isFile, err = fs.IsFile("testDir1/testFile")
+	require.NoError(t, err)
+	assert.True(t, isFile)
+	isDir, err = fs.IsDir("testDir1/testFile")
+	require.NoError(t, err)
+	assert.False(t, isDir)
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_Read(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	c, err := fs.ReadFile("testuntar/test.txt")
+	require.NoError(t, err)
+	assert.Equal(t, tarTestFileContent, string(c))
+
+	require.NoError(t, fs.Close())
+}
+
+func Test_tarFs_not_supported(t *testing.T) {
+	fs, _, err := NewTarFileSystemFromStandardFileSystem(filepath.Join("testdata", "testuntar.tar"), NoLimits())
+	require.NoError(t, err)
+	defer func() { _ = fs.Close() }()
+
+	_, err = fs.TempDir("testdata", "aaaa")
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	f, err := fs.OpenFile("testuntar/test.txt", os.O_RDWR, os.FileMode(0600))
+	defer func() {
+		if f != nil {
+			_ = f.Close()
+		}
+	}()
+	require.Error(t, err)
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	err = fs.Chmod("testuntar/test.txt", os.FileMode(0600))
+	require.Error(t, err)
+	errortest.AssertErrorDescription(t, err, "operation not permitted")
+
+	require.NoError(t, fs.Close())
+
+}
diff --git a/utils/filesystem/testdata/t.tar b/utils/filesystem/testdata/t.tar
new file mode 100644
index 0000000000000000000000000000000000000000..d5b9aa0fb5ebdcc5f85b3921202f8b4d5d513af2
GIT binary patch
literal 30720
zcmeI)OHPA800v-=(i;dfJT73oM_Y+$qAvQ_czS1y5>04i0j9xkHZ)8+!1oWkKb$VR
zyJF!{rqMM`kq%YYl;4keDvzRyp^Rl6n>Ni?p$xIuGz;I?$MTocd3)S!itW?krGM?;
zu3huD`D_2X@$;vY|G7V%?+?eY)JDhwPt3m#U7hleO=#u+7hC5?{&^!j9G?4`FP!sl
z>y+#0`ycZ3@cy?kru>^wrToj8?_+eifBydO4FUuR5FkK+009C72oNAZVD$oz1_1&D
z2oNAZfB*pk1PBlyuzG>d?oYK;_a}EQa=~MA|ECSP|JB7x|7WzXGV{qH|Ao3gU-?h&
z|CC+Izpg{0|1)#cqs#3K0t5&UAV7cs0RjXF5FkKc^#ZFeIC($<1PBlyK!5-N0t5&U
zAV6ThVC`e*8|41a#QsMaWA1;1E`R^Y_g@CW#eQrJ0t5&UAV7cs0RjXF5FkKc^#ZFe
qxOhMU1PBlyK!5-N0t5&UAV6TZSh2vu1PBlyK!5-N0t5)$oWMKSbAGu1

literal 0
HcmV?d00001

diff --git a/utils/filesystem/testdata/testuntar.tar b/utils/filesystem/testdata/testuntar.tar
new file mode 100644
index 0000000000000000000000000000000000000000..570d3077eb9be452c5fe985f80d900e8c3bba6d4
GIT binary patch
literal 6656
zcmeHL&ubGw6rNfuCDL23!U&?LraPP29q=Gtq-fNpQ1vEklSwnOyOYlDO6<Xt;GyRn
z#DfT4#G8mj5b@|iq2M22C@6UFfAGz=>4r2WN;imhW*KI7XWzH;zBk|e*qBFg62_id
zL=#IEB*qxhC8WVJeMTswmTi|T;y^pR%@B4b)}ub)Bm(C#?t5~is_}mOs}14TikqUD
zyZ<`CqvG#UhDeDymPHA%U1SmB+5}-M_nr&^JmT@Ml&6o+r?kA<T3gwE_x;paxYM}+
zS#hrx@M7GGR~sV2o`OMgm*c908AF5-j%2`b%v&+83!g{D9@;G02DGhjwmJ<cXCTi)
z)?-a$oxHqIT=`J`cx~bJ&R4bkY4OVP%x57kJ#SyUdH?o}3#bQ!&|83MetG}U26LTL
z7+I4VW{Jnx3q|1hxZ#C0Uxe$}7gZisd2z0AONp4{Kq`)T%L|&j-J3c#kACegd;m=$
ztP4n)^ObVXZn$Im?OnU;Q0uvwSq=9ivubOrJ+CU|nG?EY8m?zh{?Ip*OPT(!o)b##
zw4dy>+mZB>SjZ4Vz;G;+J2FubPD0?*QyU4#HID0^SOZR@baagF-#Gn4J0g@Y*P$+@
z1X(sM!8Bm>KbqEDTC@0P!#x+^K9I5Tr|$mz4{Ly){~4nu{(TfqtonQN`ounOfd4WB
znf`mrR{k9tuYa29pTYTn5Qo72-|YYLkHsM3s$SwlFW}Mnxxx~cYMrNLy{`B@sL60s
z4%(Re|A&D-{qJo+^Y?A6{#`rMzn!lC4r~LA{__vT0mMH)L>+)I5B^~WGW~boJm_{F
z<Ft6u?L5NW&eLw^r9NpN(!a6#&(43;VetRYw%}io(f=W7H|B-`!+>F6Bm+MIkFh<C

literal 0
HcmV?d00001