[MM-60513] Import profile pictures from Slack.

commands/transform.go

@@ -44,6 +44,7 @@ func init() {
 	TransformSlackCmd.Flags().BoolP("skip-convert-posts", "c", false, "Skips converting mentions and post markup. Only for testing purposes")
 	TransformSlackCmd.Flags().BoolP("skip-attachments", "a", false, "Skips copying the attachments from the import file")
 	TransformSlackCmd.Flags().Bool("skip-empty-emails", false, "Ignore empty email addresses from the import file. Note that this results in invalid data.")
+	TransformSlackCmd.Flags().Bool("skip-profile-pictures", false, "Skips copying the profile pictures from the import file")
 	TransformSlackCmd.Flags().String("default-email-domain", "", "If this flag is provided: When a user's email address is empty, the output's email address will be generated from their username and the provided domain.")
 	TransformSlackCmd.Flags().BoolP("allow-download", "l", false, "Allows downloading the attachments for the import file")
 	TransformSlackCmd.Flags().BoolP("discard-invalid-props", "p", false, "Skips converting posts with invalid props instead discarding the props themselves")
@@ -65,6 +66,7 @@ func transformSlackCmdF(cmd *cobra.Command, args []string) error {
 	attachmentsDir, _ := cmd.Flags().GetString("attachments-dir")
 	skipConvertPosts, _ := cmd.Flags().GetBool("skip-convert-posts")
 	skipAttachments, _ := cmd.Flags().GetBool("skip-attachments")
+	skipProfilePictures, _ := cmd.Flags().GetBool("skip-profile-pictures")
 	skipEmptyEmails, _ := cmd.Flags().GetBool("skip-empty-emails")
 	defaultEmailDomain, _ := cmd.Flags().GetString("default-email-domain")
 	allowDownload, _ := cmd.Flags().GetBool("allow-download")
@@ -93,6 +95,19 @@ func transformSlackCmdF(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if !skipProfilePictures {
+		profilePicturesDir := path.Join(attachmentsDir, "profile_pictures")
+		if fileInfo, err := os.Stat(profilePicturesDir); os.IsNotExist(err) {
+			if createErr := os.MkdirAll(profilePicturesDir, 0755); createErr != nil {
+				return createErr
+			}
+		} else if err != nil {
+			return err
+		} else if !fileInfo.IsDir() {
+			return fmt.Errorf("File \"%s\" is not a directory", attachmentsDir)
+		}
+	}
+
 	// input file
 	fileReader, err := os.Open(inputFilePath)
 	if err != nil {

services/slack/export.go

@@ -107,7 +107,7 @@ func GetImportLineFromUser(user *IntermediateUser, team string) *imports.LineImp
 		})
 	}
 
-	return &imports.LineImportData{
+	result := imports.LineImportData{
 		Type: "user",
 		User: &imports.UserImportData{
 			Username:  model.NewString(user.Username),
@@ -126,6 +126,10 @@ func GetImportLineFromUser(user *IntermediateUser, team string) *imports.LineImp
 			},
 		},
 	}
+	if len(user.ProfilePicture) > 0 {
+		result.User.ProfileImage = model.NewString(user.ProfilePicture)
+	}
+	return &result
 }
 
 func GetAttachmentImportDataFromPaths(paths []string) []imports.AttachmentImportData {

services/slack/intermediate.go

@@ -75,15 +75,16 @@ func (c *IntermediateChannel) Sanitise(logger log.FieldLogger) {
 }
 
 type IntermediateUser struct {
-	Id          string   `json:"id"`
-	Username    string   `json:"username"`
-	FirstName   string   `json:"first_name"`
-	LastName    string   `json:"last_name"`
-	Position    string   `json:"position"`
-	Email       string   `json:"email"`
-	Password    string   `json:"password"`
-	Memberships []string `json:"memberships"`
-	DeleteAt    int64    `json:"delete_at"`
+	Id             string   `json:"id"`
+	Username       string   `json:"username"`
+	FirstName      string   `json:"first_name"`
+	LastName       string   `json:"last_name"`
+	Position       string   `json:"position"`
+	Email          string   `json:"email"`
+	Password       string   `json:"password"`
+	Memberships    []string `json:"memberships"`
+	DeleteAt       int64    `json:"delete_at"`
+	ProfilePicture string   `json:"profile_picture"`
 }
 
 func (u *IntermediateUser) Sanitise(logger log.FieldLogger, defaultEmailDomain string, skipEmptyEmails bool) {
@@ -129,7 +130,7 @@ type Intermediate struct {
 	Posts           []*IntermediatePost          `json:"posts"`
 }
 
-func (t *Transformer) TransformUsers(users []SlackUser, skipEmptyEmails bool, defaultEmailDomain string) {
+func (t *Transformer) TransformUsers(users []SlackUser, skipEmptyEmails bool, defaultEmailDomain string, attachmentsDir string, pictures map[string]*zip.File) {
 	t.Logger.Info("Transforming users")
 
 	t.Logger.Debugf("TransformUsers: Input SlackUser structs: %+v", users)
@@ -165,6 +166,15 @@ func (t *Transformer) TransformUsers(users []SlackUser, skipEmptyEmails bool, de
 
 		t.Logger.Debugf("TransformUsers: newUser IntermediateUser struct: %+v", newUser)
 
+		if len(user.Profile.ImagePath) > 0 {
+			err := addProfilePicture(user.Profile.ImagePath, pictures, attachmentsDir)
+			if err == nil {
+				newUser.ProfilePicture = user.Profile.ImagePath
+			} else {
+				t.Logger.Warnf("Failed to import profile picture: %s", err.Error())
+			}
+		}
+
 		if user.IsBot {
 			newUser.Id = user.Profile.BotID
 		}
@@ -177,6 +187,33 @@ func (t *Transformer) TransformUsers(users []SlackUser, skipEmptyEmails bool, de
 	t.Intermediate.UsersById = resultUsers
 }
 
+func addProfilePicture(zipPath string, pictures map[string]*zip.File, attachmentsDir string) error {
+	zipFile, ok := pictures[zipPath]
+	if !ok {
+		return errors.Errorf("failed to retrieve profile picture with id %s", zipPath)
+	}
+
+	zipFileReader, err := zipFile.Open()
+	if err != nil {
+		return errors.Wrapf(err, "failed to open profile picture from zipfile for id %s", zipPath)
+	}
+	defer zipFileReader.Close()
+
+	destFilePath := path.Join(attachmentsDir, zipPath)
+	destFile, err := os.Create(destFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create file %s in the profile pictures directory", zipPath)
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, zipFileReader)
+	if err != nil {
+		return errors.Wrapf(err, "failed to write file %s in the profile pictures directory", zipPath)
+	}
+
+	return nil
+}
+
 func filterValidMembers(members []string, users map[string]*IntermediateUser) []string {
 	validMembers := []string{}
 	for _, member := range members {
@@ -760,7 +797,7 @@ func (t *Transformer) TransformPosts(slackExport *SlackExport, attachmentsDir st
 }
 
 func (t *Transformer) Transform(slackExport *SlackExport, attachmentsDir string, skipAttachments, discardInvalidProps, allowDownload, skipEmptyEmails bool, defaultEmailDomain string) error {
-	t.TransformUsers(slackExport.Users, skipEmptyEmails, defaultEmailDomain)
+	t.TransformUsers(slackExport.Users, skipEmptyEmails, defaultEmailDomain, attachmentsDir, slackExport.ProfilePictures)
 
 	if err := t.TransformAllChannels(slackExport); err != nil {
 		return err

services/slack/intermediate_test.go

@@ -1,6 +1,7 @@
 package slack
 
 import (
+	"archive/zip"
 	"bytes"
 	"fmt"
 	"os"
@@ -571,7 +572,8 @@ func TestTransformUsers(t *testing.T) {
 
 	defaultEmailDomain := ""
 	skipEmptyEmails := false
-	slackTransformer.TransformUsers(users, skipEmptyEmails, defaultEmailDomain)
+	pictures := make(map[string]*zip.File)
+	slackTransformer.TransformUsers(users, skipEmptyEmails, defaultEmailDomain, "data/", pictures)
 	require.Len(t, slackTransformer.Intermediate.UsersById, len(users))
 
 	for i, id := range []string{id1, id2, id3} {
@@ -641,7 +643,8 @@ func TestDeleteAt(t *testing.T) {
 
 	defaultEmailDomain := ""
 	skipEmptyEmails := false
-	slackTransformer.TransformUsers(users, skipEmptyEmails, defaultEmailDomain)
+	pictures := make(map[string]*zip.File)
+	slackTransformer.TransformUsers(users, skipEmptyEmails, defaultEmailDomain, "data/", pictures)
 	require.Zero(t, slackTransformer.Intermediate.UsersById[activeUsers[0].Id].DeleteAt)
 	require.Zero(t, slackTransformer.Intermediate.UsersById[activeUsers[1].Id].DeleteAt)
 	require.NotZero(t, slackTransformer.Intermediate.UsersById[inactiveUsers[0].Id].DeleteAt)

services/slack/parse.go

@@ -28,10 +28,11 @@ type SlackChannelSub struct {
 }
 
 type SlackProfile struct {
-	BotID    string `json:"bot_id"`
-	RealName string `json:"real_name"`
-	Email    string `json:"email"`
-	Title    string `json:"title"`
+	BotID     string `json:"bot_id"`
+	RealName  string `json:"real_name"`
+	Email     string `json:"email"`
+	Title     string `json:"title"`
+	ImagePath string `json:"image_path"`
 }
 
 type SlackUser struct {
@@ -134,6 +135,7 @@ type SlackExport struct {
 	Users           []SlackUser
 	Posts           map[string][]SlackPost
 	Uploads         map[string]*zip.File
+	ProfilePictures map[string]*zip.File
 }
 
 func (t *Transformer) SlackParseUsers(data io.Reader) ([]SlackUser, error) {
@@ -345,6 +347,7 @@ func (t *Transformer) ParseSlackExportFile(zipReader *zip.Reader, skipConvertPos
 	slackExport := SlackExport{TeamName: t.TeamName}
 	slackExport.Posts = make(map[string][]SlackPost)
 	slackExport.Uploads = make(map[string]*zip.File)
+	slackExport.ProfilePictures = make(map[string]*zip.File)
 	numFiles := len(zipReader.File)
 
 	for i, file := range zipReader.File {
@@ -393,6 +396,8 @@ func (t *Transformer) ParseSlackExportFile(zipReader *zip.Reader, skipConvertPos
 					}
 				} else if len(spl) == 3 && spl[0] == "__uploads" {
 					slackExport.Uploads[spl[1]] = file
+				} else if len(spl) == 2 && spl[0] == "profile_pictures" {
+					slackExport.ProfilePictures[file.Name] = file
 				}
 			}