Import Slack bot_message type posts. (#4261)

This includes all messages from integrations, as far as I can tell.

Messages are "owned" by a special user that is deactivated once the
import completes.

Override User Names are only shown where the individual Slack posts have
a username override in them. Ones set centrally through the Slack Web
Hooks administration, or by Slack-official integrations, aren't known so
we can't set them on the imported posts. Same for icons.

The attachment "colors" aren't imported as Mattermost does not appear to
have any equivalent feature.

Fixes PLT-48

3 files changed, 171 insertions, 15 deletions

diff --git a/api/import.go b/api/import.go
index 51c2ef20f..3ac6a9ce9 100644
--- a/api/import.go
+++ b/api/import.go
@@ -6,6 +6,7 @@ package api
 import (
 	"bytes"
 	"io"
+	"regexp"
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/platform/model"
@@ -80,3 +81,57 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string,
 
 	return fileInfo, nil
 }
+
+func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
+	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
+	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
+
+	post.AddProp("from_webhook", "true")
+
+	if _, ok := props["override_username"]; !ok {
+		post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME)
+	}
+
+	if len(props) > 0 {
+		for key, val := range props {
+			if key == "attachments" {
+				if list, success := val.([]interface{}); success {
+					// parse attachment links into Markdown format
+					for i, aInt := range list {
+						attachment := aInt.(map[string]interface{})
+						if aText, ok := attachment["text"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["text"] = aText
+							list[i] = attachment
+						}
+						if aText, ok := attachment["pretext"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["pretext"] = aText
+							list[i] = attachment
+						}
+						if fVal, ok := attachment["fields"]; ok {
+							if fields, ok := fVal.([]interface{}); ok {
+								// parse attachment field links into Markdown format
+								for j, fInt := range fields {
+									field := fInt.(map[string]interface{})
+									if fValue, ok := field["value"].(string); ok {
+										fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})")
+										field["value"] = fValue
+										fields[j] = field
+									}
+								}
+								attachment["fields"] = fields
+								list[i] = attachment
+							}
+						}
+					}
+					post.AddProp(key, list)
+				}
+			} else if key != "from_webhook" {
+				post.AddProp(key, val)
+			}
+		}
+	}
+
+	ImportPost(post)
+}
diff --git a/api/slackimport.go b/api/slackimport.go
index af27b0d30..f6ee9fc4c 100644
--- a/api/slackimport.go
+++ b/api/slackimport.go
@@ -38,16 +38,17 @@ type SlackFile struct {
 }
 
 type SlackPost struct {
-	User        string        `json:"user"`
-	BotId       string        `json:"bot_id"`
-	BotUsername string        `json:"username"`
-	Text        string        `json:"text"`
-	TimeStamp   string        `json:"ts"`
-	Type        string        `json:"type"`
-	SubType     string        `json:"subtype"`
-	Comment     *SlackComment `json:"comment"`
-	Upload      bool          `json:"upload"`
-	File        *SlackFile    `json:"file"`
+	User        string            `json:"user"`
+	BotId       string            `json:"bot_id"`
+	BotUsername string            `json:"username"`
+	Text        string            `json:"text"`
+	TimeStamp   string            `json:"ts"`
+	Type        string            `json:"type"`
+	SubType     string            `json:"subtype"`
+	Comment     *SlackComment     `json:"comment"`
+	Upload      bool              `json:"upload"`
+	File        *SlackFile        `json:"file"`
+	Attachments []SlackAttachment `json:"attachments"`
 }
 
 type SlackComment struct {
@@ -55,6 +56,13 @@ type SlackComment struct {
 	Comment string `json:"comment"`
 }
 
+type SlackAttachment struct {
+	Id      int                      `json:"id"`
+	Text    string                   `json:"text"`
+	Pretext string                   `json:"pretext"`
+	Fields  []map[string]interface{} `json:"fields"`
+}
+
 func SlackConvertTimeStamp(ts string) int64 {
 	timeString := strings.SplitN(ts, ".", 2)[0]
 
@@ -167,7 +175,37 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
 	return addedUsers
 }
 
-func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File) {
+func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
+	var team *model.Team
+	if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
+		log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
+		return nil
+	} else {
+		team = result.Data.(*model.Team)
+	}
+
+	password := model.NewId()
+	username := "slackimportuser_" + model.NewId()
+	email := username + "@localhost"
+
+	botUser := model.User{
+		Username:  username,
+		FirstName: "",
+		LastName:  "",
+		Email:     email,
+		Password:  password,
+	}
+
+	if mUser := ImportUser(team, &botUser); mUser != nil {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
+		return mUser
+	} else {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
+		return nil
+	}
+}
+
+func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
 	for _, sPost := range posts {
 		switch {
 		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
@@ -216,7 +254,37 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
 			}
 			ImportPost(&newPost)
 		case sPost.Type == "message" && sPost.SubType == "bot_message":
-			continue
+			if botUser == nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
+				continue
+			} else if sPost.BotId == "" {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn"))
+				continue
+			}
+
+			props := make(model.StringInterface)
+			props["override_username"] = sPost.BotUsername
+			if len(sPost.Attachments) > 0 {
+				var mAttachments []interface{}
+				for _, attachment := range sPost.Attachments {
+					mAttachments = append(mAttachments, map[string]interface{}{
+						"text":    attachment.Text,
+						"pretext": attachment.Pretext,
+						"fields":  attachment.Fields,
+					})
+				}
+				props["attachments"] = mAttachments
+			}
+
+			post := &model.Post{
+				UserId:    botUser.Id,
+				ChannelId: channel.Id,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Message:   sPost.Text,
+				Type:      model.POST_SLACK_ATTACHMENT,
+			}
+
+			ImportIncomingWebhookPost(post, props)
 		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
 			if sPost.User == "" {
 				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -281,6 +349,13 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
 	}
 }
 
+func deactivateSlackBotUser(user *model.User) {
+	_, err := UpdateActive(user, false)
+	if err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
+	}
+}
+
 func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
 	for _, member := range members {
 		if user, ok := users[member]; !ok {
@@ -293,7 +368,7 @@ func addSlackUsersToChannel(members []string, users map[string]*model.User, chan
 	}
 }
 
-func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, log *bytes.Buffer) map[string]*model.Channel {
+func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel {
 	// Write Header
 	log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
 	log.WriteString("=================\r\n\r\n")
@@ -323,7 +398,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str
 		addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
 		log.WriteString(newChannel.DisplayName + "\r\n")
 		addedChannels[sChannel.Id] = mChannel
-		SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads)
+		SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
 	}
 
 	return addedChannels
@@ -424,7 +499,13 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model
 	posts = SlackConvertChannelMentions(channels, posts)
 
 	addedUsers := SlackAddUsers(teamID, users, log)
-	SlackAddChannels(teamID, channels, posts, addedUsers, uploads, log)
+	botUser := SlackAddBotUser(teamID, log)
+
+	SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
+
+	if botUser != nil {
+		deactivateSlackBotUser(botUser)
+	}
 
 	log.WriteString(utils.T("api.slackimport.slack_import.notes"))
 	log.WriteString("=======\r\n\r\n")
diff --git a/i18n/en.json b/i18n/en.json
index bb4b61168..fe7de229e 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -1548,6 +1548,10 @@
     "translation": "Slack bot posts are not imported yet"
   },
   {
+    "id": "api.slackimport.slack_add_posts.bot_user_no_exists.warn",
+    "translation": "Slack Importer: Not importing bot message as the bot-importing user does not exist."
+  },
+  {
     "id": "api.slackimport.slack_add_posts.msg_no_comment.debug",
     "translation": "File comment undefined"
   },
@@ -1556,6 +1560,10 @@
     "translation": "Message without user"
   },
   {
+    "id": "api.slackimport.slack_add_posts.no_bot_id.warn",
+    "translation": "Slack Importer: Not importing bot message due to lack of BotId field."
+  },
+  {
     "id": "api.slackimport.slack_add_posts.unsupported.warn",
     "translation": "Unsupported post type: %v, %v"
   },
@@ -1600,6 +1608,14 @@
     "translation": "Unable to import user: {{.Username}}\r\n"
   },
   {
+    "id": "api.slackimport.slack_add_bot_user.email_pwd",
+    "translation": "Slack Bot/Integration Posts Import User: Email, Password: {{.Email}}, {{.Password}}\r\n"
+  },
+  {
+    "id": "api.slackimport.slack_add_bot_user.unable_import",
+    "translation": "Unable to import Slack Bot/Integration Posts Import User: {{.Username}}\r\n"
+  },
+  {
     "id": "api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn",
     "translation": "Failed to compile the !channel matching regular expression for Slack channel {{.ChannelID}} {{.ChannelName}}"
   },
@@ -1612,6 +1628,10 @@
     "translation": "Failed to compile the @mention matching regular expression for Slack user {{.UserID}} {{.Username}}"
   },
   {
+    "id": "api.slackimport.slack_deactivate_bot_user.failed_to_deactivate",
+    "translation": "Slack Importer: Failed to deactivate the bot-importing user."
+  },
+  {
     "id": "api.slackimport.slack_import.log",
     "translation": "Mattermost Slack Import Log\r\n"
   },