From 5ca05124cf2c4f106f0fef95546bd4b9d376af3e Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Wed, 12 Oct 2016 14:31:05 +0100 Subject: Import Slack uploads if present in zip archive. (#4088) * Import Slack uploads if present in zip archive. This is part 3 of PLT-4280, to support importing file uploads when importing from Slack. It is assumed the uploads in the zip archive will be present as per the output of slack-advanced-exporter: https://github.com/grundleborg/slack-advanced-exporter If the uploads are not present (ie. this is a vanilla Slack export archive) uploads are treated in the same way as before this patch, providing only a link to the upload on Slack's servers. * Update to new Files API. --- api/import.go | 28 ++++++++++++++++++++++++++ api/slackimport.go | 59 +++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 5 deletions(-) (limited to 'api') diff --git a/api/import.go b/api/import.go index 59d098d93..51c2ef20f 100644 --- a/api/import.go +++ b/api/import.go @@ -4,6 +4,9 @@ package api import ( + "bytes" + "io" + l4g "github.com/alecthomas/log4go" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" @@ -52,3 +55,28 @@ func ImportChannel(channel *model.Channel) *model.Channel { return sc } } + +func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { + buf := bytes.NewBuffer(nil) + io.Copy(buf, file) + data := buf.Bytes() + + previewPathList := []string{} + thumbnailPathList := []string{} + imageDataList := [][]byte{} + + fileInfo, err := doUploadFile(teamId, channelId, userId, fileName, data) + if err != nil { + return nil, err + } + + if fileInfo.PreviewPath != "" || fileInfo.ThumbnailPath != "" { + previewPathList = append(previewPathList, fileInfo.PreviewPath) + thumbnailPathList = append(thumbnailPathList, fileInfo.ThumbnailPath) + imageDataList = append(imageDataList, data) + } + + go handleImages(previewPathList, thumbnailPathList, imageDataList) + + return fileInfo, nil +} diff --git a/api/slackimport.go b/api/slackimport.go index 59d5f621a..688d25370 100644 --- a/api/slackimport.go +++ b/api/slackimport.go @@ -12,6 +12,7 @@ import ( "github.com/mattermost/platform/utils" "io" "mime/multipart" + "path/filepath" "regexp" "strconv" "strings" @@ -31,6 +32,11 @@ type SlackUser struct { Profile map[string]string `json:"profile"` } +type SlackFile struct { + Id string `json:"id"` + Title string `json:"title"` +} + type SlackPost struct { User string `json:"user"` BotId string `json:"bot_id"` @@ -40,6 +46,8 @@ type SlackPost struct { Type string `json:"type"` SubType string `json:"subtype"` Comment *SlackComment `json:"comment"` + Upload bool `json:"upload"` + File *SlackFile `json:"file"` } type SlackComment struct { @@ -159,7 +167,7 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map return addedUsers } -func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]*model.User) { +func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File) { for _, sPost := range posts { switch { case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): @@ -176,7 +184,19 @@ func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]* Message: sPost.Text, CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), } + if sPost.Upload { + if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true { + newPost.FileIds = append(newPost.FileIds, fileInfo.Id) + newPost.Message = sPost.File.Title + } + } ImportPost(&newPost) + for _, fileId := range newPost.FileIds { + if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil { + l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err) + } + } + case sPost.Type == "message" && sPost.SubType == "file_comment": if sPost.Comment == nil { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug")) @@ -219,6 +239,33 @@ func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]* } } +func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) { + if sPost.File != nil { + if file, ok := uploads[sPost.File.Id]; ok == true { + openFile, err := file.Open() + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) + return nil, false + } + defer openFile.Close() + + uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) + return nil, false + } + + return uploadedFile, true + } else { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id})) + return nil, false + } + } else { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn")) + return nil, false + } +} + func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { for _, member := range members { if user, ok := users[member]; !ok { @@ -231,7 +278,7 @@ func addSlackUsersToChannel(members []string, users map[string]*model.User, chan } } -func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, log *bytes.Buffer) map[string]*model.Channel { +func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, log *bytes.Buffer) map[string]*model.Channel { // Write Header log.WriteString(utils.T("api.slackimport.slack_add_channels.added")) log.WriteString("=================\r\n\r\n") @@ -261,7 +308,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str addSlackUsersToChannel(sChannel.Members, users, mChannel, log) log.WriteString(newChannel.DisplayName + "\r\n") addedChannels[sChannel.Id] = mChannel - SlackAddPosts(mChannel, posts[sChannel.Name], users) + SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads) } return addedChannels @@ -331,6 +378,7 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model var channels []SlackChannel var users []SlackUser posts := make(map[string][]SlackPost) + uploads := make(map[string]*zip.File) for _, file := range zipreader.File { reader, err := file.Open() if err != nil { @@ -351,8 +399,9 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model } else { posts[channel] = append(posts[channel], newposts...) } + } else if len(spl) == 3 && spl[0] == "__uploads" { + uploads[spl[1]] = file } - } } @@ -360,7 +409,7 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model posts = SlackConvertChannelMentions(channels, posts) addedUsers := SlackAddUsers(teamID, users, log) - SlackAddChannels(teamID, channels, posts, addedUsers, log) + SlackAddChannels(teamID, channels, posts, addedUsers, uploads, log) log.WriteString(utils.T("api.slackimport.slack_import.notes")) log.WriteString("=======\r\n\r\n") -- cgit v1.2.3-1-g7c22