Import Slack bot_message type posts. (#4261)

This includes all messages from integrations, as far as I can tell.

Messages are "owned" by a special user that is deactivated once the
import completes.

Override User Names are only shown where the individual Slack posts have
a username override in them. Ones set centrally through the Slack Web
Hooks administration, or by Slack-official integrations, aren't known so
we can't set them on the imported posts. Same for icons.

The attachment "colors" aren't imported as Mattermost does not appear to
have any equivalent feature.

Fixes PLT-48

2 files changed, 151 insertions, 15 deletions

diff --git a/api/import.go b/api/import.go
index 51c2ef20f..3ac6a9ce9 100644
--- a/api/import.go
+++ b/api/import.go
@@ -6,6 +6,7 @@ package api
 import (
 	"bytes"
 	"io"
+	"regexp"
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/platform/model"
@@ -80,3 +81,57 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string,
 
 	return fileInfo, nil
 }
+
+func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
+	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
+	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
+
+	post.AddProp("from_webhook", "true")
+
+	if _, ok := props["override_username"]; !ok {
+		post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME)
+	}
+
+	if len(props) > 0 {
+		for key, val := range props {
+			if key == "attachments" {
+				if list, success := val.([]interface{}); success {
+					// parse attachment links into Markdown format
+					for i, aInt := range list {
+						attachment := aInt.(map[string]interface{})
+						if aText, ok := attachment["text"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["text"] = aText
+							list[i] = attachment
+						}
+						if aText, ok := attachment["pretext"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["pretext"] = aText
+							list[i] = attachment
+						}
+						if fVal, ok := attachment["fields"]; ok {
+							if fields, ok := fVal.([]interface{}); ok {
+								// parse attachment field links into Markdown format
+								for j, fInt := range fields {
+									field := fInt.(map[string]interface{})
+									if fValue, ok := field["value"].(string); ok {
+										fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})")
+										field["value"] = fValue
+										fields[j] = field
+									}
+								}
+								attachment["fields"] = fields
+								list[i] = attachment
+							}
+						}
+					}
+					post.AddProp(key, list)
+				}
+			} else if key != "from_webhook" {
+				post.AddProp(key, val)
+			}
+		}
+	}
+
+	ImportPost(post)
+}
diff --git a/api/slackimport.go b/api/slackimport.go
index af27b0d30..f6ee9fc4c 100644
--- a/api/slackimport.go
+++ b/api/slackimport.go
@@ -38,16 +38,17 @@ type SlackFile struct {
 }
 
 type SlackPost struct {
-	User        string        `json:"user"`
-	BotId       string        `json:"bot_id"`
-	BotUsername string        `json:"username"`
-	Text        string        `json:"text"`
-	TimeStamp   string        `json:"ts"`
-	Type        string        `json:"type"`
-	SubType     string        `json:"subtype"`
-	Comment     *SlackComment `json:"comment"`
-	Upload      bool          `json:"upload"`
-	File        *SlackFile    `json:"file"`
+	User        string            `json:"user"`
+	BotId       string            `json:"bot_id"`
+	BotUsername string            `json:"username"`
+	Text        string            `json:"text"`
+	TimeStamp   string            `json:"ts"`
+	Type        string            `json:"type"`
+	SubType     string            `json:"subtype"`
+	Comment     *SlackComment     `json:"comment"`
+	Upload      bool              `json:"upload"`
+	File        *SlackFile        `json:"file"`
+	Attachments []SlackAttachment `json:"attachments"`
 }
 
 type SlackComment struct {
@@ -55,6 +56,13 @@ type SlackComment struct {
 	Comment string `json:"comment"`
 }
 
+type SlackAttachment struct {
+	Id      int                      `json:"id"`
+	Text    string                   `json:"text"`
+	Pretext string                   `json:"pretext"`
+	Fields  []map[string]interface{} `json:"fields"`
+}
+
 func SlackConvertTimeStamp(ts string) int64 {
 	timeString := strings.SplitN(ts, ".", 2)[0]
 
@@ -167,7 +175,37 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
 	return addedUsers
 }
 
-func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File) {
+func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
+	var team *model.Team
+	if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
+		log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
+		return nil
+	} else {
+		team = result.Data.(*model.Team)
+	}
+
+	password := model.NewId()
+	username := "slackimportuser_" + model.NewId()
+	email := username + "@localhost"
+
+	botUser := model.User{
+		Username:  username,
+		FirstName: "",
+		LastName:  "",
+		Email:     email,
+		Password:  password,
+	}
+
+	if mUser := ImportUser(team, &botUser); mUser != nil {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
+		return mUser
+	} else {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
+		return nil
+	}
+}
+
+func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
 	for _, sPost := range posts {
 		switch {
 		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
@@ -216,7 +254,37 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
 			}
 			ImportPost(&newPost)
 		case sPost.Type == "message" && sPost.SubType == "bot_message":
-			continue
+			if botUser == nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
+				continue
+			} else if sPost.BotId == "" {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn"))
+				continue
+			}
+
+			props := make(model.StringInterface)
+			props["override_username"] = sPost.BotUsername
+			if len(sPost.Attachments) > 0 {
+				var mAttachments []interface{}
+				for _, attachment := range sPost.Attachments {
+					mAttachments = append(mAttachments, map[string]interface{}{
+						"text":    attachment.Text,
+						"pretext": attachment.Pretext,
+						"fields":  attachment.Fields,
+					})
+				}
+				props["attachments"] = mAttachments
+			}
+
+			post := &model.Post{
+				UserId:    botUser.Id,
+				ChannelId: channel.Id,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Message:   sPost.Text,
+				Type:      model.POST_SLACK_ATTACHMENT,
+			}
+
+			ImportIncomingWebhookPost(post, props)
 		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
 			if sPost.User == "" {
 				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -281,6 +349,13 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
 	}
 }
 
+func deactivateSlackBotUser(user *model.User) {
+	_, err := UpdateActive(user, false)
+	if err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
+	}
+}
+
 func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
 	for _, member := range members {
 		if user, ok := users[member]; !ok {
@@ -293,7 +368,7 @@ func addSlackUsersToChannel(members []string, users map[string]*model.User, chan
 	}
 }
 
-func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, log *bytes.Buffer) map[string]*model.Channel {
+func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel {
 	// Write Header
 	log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
 	log.WriteString("=================\r\n\r\n")
@@ -323,7 +398,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str
 		addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
 		log.WriteString(newChannel.DisplayName + "\r\n")
 		addedChannels[sChannel.Id] = mChannel
-		SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads)
+		SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
 	}
 
 	return addedChannels
@@ -424,7 +499,13 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model
 	posts = SlackConvertChannelMentions(channels, posts)
 
 	addedUsers := SlackAddUsers(teamID, users, log)
-	SlackAddChannels(teamID, channels, posts, addedUsers, uploads, log)
+	botUser := SlackAddBotUser(teamID, log)
+
+	SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
+
+	if botUser != nil {
+		deactivateSlackBotUser(botUser)
+	}
 
 	log.WriteString(utils.T("api.slackimport.slack_import.notes"))
 	log.WriteString("=======\r\n\r\n")