Introduce support for Slack Enterprise Grid

.gitignore

@@ -1,2 +1,11 @@
 mmetl
-build/
+build/
+slackData
+mmetl.md5.txt
+tmp
+.DS_Store
+
+*.log
+*.zip
+*.jsonl
+.idea

commands/grid_transform.go

@@ -0,0 +1,141 @@
+package commands
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"os"
+
+	"github.com/mattermost/mmetl/services/slack"
+	"github.com/mattermost/mmetl/services/slack_grid"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var GridTransformCmd = &cobra.Command{
+	Use:   "grid-transform",
+	Short: "Transforms a slack enterprise grid into multiple workspace export files.",
+	Long:  "Accepts a Slack Enterprise Grid export file and transforms it into multiple workspace export files to be imported seperatly into Mattermost.",
+	Args:  cobra.NoArgs,
+	RunE:  gridTransformCmdF,
+}
+
+func init() {
+	GridTransformCmd.Flags().StringP("file", "f", "", "the Slack export file to clean")
+	GridTransformCmd.Flags().StringP("teamMap", "t", "", "The team mapping file to use")
+
+	GridTransformCmd.Flags().Bool("debug", true, "Whether to show debug logs or not")
+
+	if err := GridTransformCmd.MarkFlagRequired("file"); err != nil {
+		panic(err)
+	}
+
+	if err := GridTransformCmd.MarkFlagRequired("teamMap"); err != nil {
+		panic(err)
+	}
+
+	RootCmd.AddCommand(
+		GridTransformCmd,
+	)
+}
+
+func gridTransformCmdF(cmd *cobra.Command, args []string) error {
+	inputFilePath, _ := cmd.Flags().GetString("file")
+	teamMap, _ := cmd.Flags().GetString("teamMap")
+
+	debug, _ := cmd.Flags().GetBool("debug")
+
+	logger := log.New()
+	logFile, err := os.OpenFile("grid-transform-slack.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		logger.Error("error creating zip reader: %w", err)
+		return err
+	}
+	defer logFile.Close()
+	logger.SetOutput(logFile)
+	logger.SetFormatter(customLogFormatter)
+	logger.SetReportCaller(true)
+
+	if debug {
+		logger.Level = log.DebugLevel
+		logger.Info("Debug mode enabled")
+	}
+
+	// input file
+	fileReader, err := os.Open(inputFilePath)
+	if err != nil {
+		logger.Error("error opening input file: %w", err)
+		return err
+	}
+	defer fileReader.Close()
+
+	zipFileInfo, err := fileReader.Stat()
+	if err != nil {
+		logger.Error("error getting file info: %w", err)
+		return err
+	}
+
+	zipReader, err := zip.NewReader(fileReader, zipFileInfo.Size())
+	if err != nil || zipReader.File == nil {
+		logger.Error("error reading zip file %w", err)
+		return err
+	}
+
+	// we do not need a team name here.
+	slackTransformer := slack_grid.NewGridTransformer(logger)
+	teamMapFile, err := os.Open(teamMap)
+	if err != nil {
+		logger.Error("error parsing teams.json: %w", err)
+		return err
+	}
+	defer teamMapFile.Close()
+
+	teamMapDecoder := json.NewDecoder(teamMapFile)
+	err = teamMapDecoder.Decode(&slackTransformer.Teams)
+	if err != nil {
+		logger.Error("error parsing teams.json: %w", err)
+		return err
+	}
+
+	valid := slackTransformer.GridPreCheck(zipReader)
+	if !valid {
+		return nil
+	}
+
+	err = slackTransformer.ExtractDirectory(zipReader)
+	if err != nil {
+		logger.Error("error extracting zip file. error:", err)
+		return nil
+	}
+
+	slackExport, err := slackTransformer.ParseGridSlackExportFile(zipReader)
+	if err != nil {
+		logger.Error("error parsing slack export: %w", err)
+		return err
+	}
+
+	channelTypes := []struct {
+		channels []slack.SlackChannel
+		fileType slack_grid.ChannelFiles
+	}{
+		{slackExport.Public, slack_grid.ChannelFilePublic},
+		{slackExport.Private, slack_grid.ChannelFilePrivate},
+		{slackExport.GMs, slack_grid.ChannelFileGM},
+		{slackExport.DMs, slack_grid.ChannelFileDM},
+	}
+
+	for _, ct := range channelTypes {
+		err = slackTransformer.HandleMovingChannels(ct.channels, ct.fileType)
+		if err != nil {
+			logger.Errorf("Error moving %v channels: %v", ct.fileType, err)
+			return err
+		}
+	}
+
+	err = slackTransformer.ZipTeamDirectories()
+	if err != nil {
+		logger.Error("error zipping team directories", err)
+		return err
+	}
+
+	return nil
+}

services/slack/precheck.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-func (t *Transformer) checkForRequiredFile(zipReader *zip.Reader, fileName string) bool {
+func (t *Transformer) CheckForRequiredFile(zipReader *zip.Reader, fileName string) bool {
 	found := false
 	foundInSubdirectory := false
 
@@ -39,7 +39,7 @@ func (t *Transformer) Precheck(zipReader *zip.Reader) bool {
 	valid := true
 
 	for _, fileName := range requiredFiles {
-		fileExists := t.checkForRequiredFile(zipReader, fileName)
+		fileExists := t.CheckForRequiredFile(zipReader, fileName)
 
 		valid = valid && fileExists
 	}

services/slack_grid/extract.go

@@ -0,0 +1,201 @@
+package slack_grid
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+const DefaultDirPath = "tmp/slack_grid"
+
+func (t *GridTransformer) GetWorkingDir() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", errors.Wrap(err, "error getting current working directory")
+	}
+	return dir, nil
+}
+
+func (t *GridTransformer) readDir(dest string) ([]fs.DirEntry, error) {
+	files, err := os.ReadDir(dest)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("error reading directory %v", dest))
+	}
+	return files, nil
+}
+
+func (t *GridTransformer) dirHasContent(dest string) (bool, error) {
+
+	err := os.MkdirAll(t.dirPath, os.ModePerm)
+	if err != nil {
+		return false, errors.Wrap(err, "error creating directory")
+	}
+
+	entries, err := os.ReadDir(dest)
+	if err != nil {
+		return false, errors.Wrap(err, "error reading directory")
+	}
+
+	if len(entries) > 0 {
+		fmt.Printf("directory %s is not empty. Using existing data. \n", dest)
+		return true, nil
+	}
+	return false, nil
+}
+
+func (t *GridTransformer) ExtractDirectory(zipReader *zip.Reader) error {
+	t.Logger.Info("Extracting files...")
+	pwd, err := t.GetWorkingDir()
+
+	if err != nil {
+		return errors.Errorf("error getting current working directory: %v", err)
+	}
+	t.pwd = pwd
+	t.dirPath = filepath.Join(pwd, DefaultDirPath)
+	t.Logger.Infof("Extracting to %s", t.dirPath)
+
+	yes, err := t.dirHasContent(t.dirPath)
+	if err != nil {
+		return errors.Errorf("error seeing if directory has content already. %v", err)
+	}
+
+	if yes {
+		t.Logger.Infof("content exists in the directory %s. Skipping extraction.", t.dirPath)
+		return nil
+	}
+
+	totalFiles := len(zipReader.File)
+
+	for i, f := range zipReader.File {
+
+		// Slack file conversations have a : in the name. So, "FC:123:123" would be valid. This simply removes the : from the name.
+		// currently, these imports are not supported by the slack grid importer so the files are not referenced later.
+		sanitizedFileName := strings.ReplaceAll(f.Name, ":", "")
+		fpath := filepath.Join(t.dirPath, sanitizedFileName)
+
+		if f.FileInfo().IsDir() {
+			// Make Folder
+			err := os.MkdirAll(fpath, os.ModePerm)
+			if err != nil {
+				return errors.Wrap(err, "error creating directory")
+			}
+			continue
+		}
+
+		// Make File
+		if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+			return errors.Wrap(err, "error creating directory")
+		}
+
+		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			return errors.Wrap(err, "error creating files")
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			outFile.Close()
+			return errors.Wrap(err, "error opening files")
+		}
+
+		_, err = io.Copy(outFile, rc)
+
+		// Close the file without defer to close before next iteration of loop
+		outFile.Close()
+		rc.Close()
+		if i%1000 == 0 || i == totalFiles-1 {
+			t.Logger.Infof("Extracting file %d of %d", i, totalFiles)
+		}
+		if err != nil {
+			return errors.Wrap(err, "error copying files")
+		}
+	}
+	t.Logger.Info("Finished extracting files")
+
+	return nil
+}
+
+func (t *GridTransformer) ZipTeamDirectories() error {
+
+	// zip the directories under /teams
+
+	teams, err := t.readDir(filepath.Join(t.dirPath, "teams"))
+
+	t.Logger.Infof("Zipping %v team directories...", len(teams))
+
+	if err != nil {
+		return errors.Wrap(err, "error reading teams directory")
+	}
+	// provide each as a root level export
+
+	for _, team := range teams {
+		teamPath := filepath.Join(t.dirPath, "teams", team.Name())
+		teamZipPath := filepath.Join(t.pwd, team.Name()+".zip")
+
+		t.Logger.Infof("Zipping %s to %s", teamPath, teamZipPath)
+
+		err := ZipDir(teamPath, teamZipPath)
+		if err != nil {
+			return errors.Wrap(err, "error zipping team directory")
+		}
+	}
+	// zip the remaining files up and provide a "leftovers" zip file.
+
+	return nil
+}
+
+func ZipDir(source, target string) error {
+	zipfile, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer zipfile.Close()
+
+	archive := zip.NewWriter(zipfile)
+	defer archive.Close()
+
+	err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+
+		header.Name = filepath.Join(".", path[len(source):])
+
+		if info.IsDir() {
+			header.Name += "/"
+		} else {
+			header.Method = zip.Deflate
+		}
+
+		writer, err := archive.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+			_, err = io.Copy(writer, file)
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	})
+
+	return err
+}