summaryrefslogtreecommitdiffstats
path: root/api
diff options
context:
space:
mode:
authorGeorge Goldberg <george@gberg.me>2017-01-20 14:47:14 +0000
committerChristopher Speller <crspeller@gmail.com>2017-01-20 09:47:14 -0500
commit2de6c5394ec3a1cd974aae46c41f61fb0e9f9bd8 (patch)
tree47dd540e415b75c0a61b8beeb1e7f6856cc8b361 /api
parent6097f93704862215791ce8855c31471e40ef0af1 (diff)
downloadchat-2de6c5394ec3a1cd974aae46c41f61fb0e9f9bd8.tar.gz
chat-2de6c5394ec3a1cd974aae46c41f61fb0e9f9bd8.tar.bz2
chat-2de6c5394ec3a1cd974aae46c41f61fb0e9f9bd8.zip
Move Slack Import to App Layer. (#5135)
Diffstat (limited to 'api')
-rw-r--r--api/file.go191
-rw-r--r--api/import.go158
-rw-r--r--api/slackimport.go622
-rw-r--r--api/slackimport_test.go240
-rw-r--r--api/team.go2
5 files changed, 3 insertions, 1210 deletions
diff --git a/api/file.go b/api/file.go
index fdc9a8747..9fda76d8f 100644
--- a/api/file.go
+++ b/api/file.go
@@ -5,49 +5,20 @@ package api
import (
"bytes"
- "image"
- "image/color"
- "image/draw"
_ "image/gif"
- "image/jpeg"
"io"
"net/http"
"net/url"
- "path/filepath"
"strconv"
- "strings"
l4g "github.com/alecthomas/log4go"
- "github.com/disintegration/imaging"
"github.com/gorilla/mux"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
- "github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
)
-const (
- /*
- EXIF Image Orientations
- 1 2 3 4 5 6 7 8
-
- 888888 888888 88 88 8888888888 88 88 8888888888
- 88 88 88 88 88 88 88 88 88 88 88 88
- 8888 8888 8888 8888 88 8888888888 8888888888 88
- 88 88 88 88
- 88 88 888888 888888
- */
- Upright = 1
- UprightMirrored = 2
- UpsideDown = 3
- UpsideDownMirrored = 4
- RotatedCWMirrored = 5
- RotatedCCW = 6
- RotatedCCWMirrored = 7
- RotatedCW = 8
-)
-
func InitFile() {
l4g.Debug(utils.T("api.file.init.debug"))
@@ -119,7 +90,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
io.Copy(buf, file)
data := buf.Bytes()
- info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
+ info, err := app.DoUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
if err != nil {
c.Err = err
return
@@ -138,169 +109,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
}
- handleImages(previewPathList, thumbnailPathList, imageDataList)
+ app.HandleImages(previewPathList, thumbnailPathList, imageDataList)
w.Write([]byte(resStruct.ToJson()))
}
-func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
- filename := filepath.Base(rawFilename)
-
- info, err := model.GetInfoForBytes(filename, data)
- if err != nil {
- err.StatusCode = http.StatusBadRequest
- return nil, err
- }
-
- info.Id = model.NewId()
- info.CreatorId = userId
-
- pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
- info.Path = pathPrefix + filename
-
- if info.IsImage() {
- // Check dimensions before loading the whole thing into memory later on
- if info.Width*info.Height > model.MaxImageSize {
- err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
- err.StatusCode = http.StatusBadRequest
- return nil, err
- }
-
- nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
- info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
- info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
- }
-
- if err := app.WriteFile(data, info.Path); err != nil {
- return nil, err
- }
-
- if result := <-app.Srv.Store.FileInfo().Save(info); result.Err != nil {
- return nil, result.Err
- }
-
- return info, nil
-}
-
-func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
- for i, data := range fileData {
- go func(i int, data []byte) {
- img, width, height := prepareImage(fileData[i])
- if img != nil {
- go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
- go generatePreviewImage(*img, previewPathList[i], width)
- }
- }(i, data)
- }
-}
-
-func prepareImage(fileData []byte) (*image.Image, int, int) {
- // Decode image bytes into Image object
- img, imgType, err := image.Decode(bytes.NewReader(fileData))
- if err != nil {
- l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
- return nil, 0, 0
- }
-
- width := img.Bounds().Dx()
- height := img.Bounds().Dy()
-
- // Fill in the background of a potentially-transparent png file as white
- if imgType == "png" {
- dst := image.NewRGBA(img.Bounds())
- draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
- draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
- img = dst
- }
-
- // Flip the image to be upright
- orientation, _ := getImageOrientation(fileData)
-
- switch orientation {
- case UprightMirrored:
- img = imaging.FlipH(img)
- case UpsideDown:
- img = imaging.Rotate180(img)
- case UpsideDownMirrored:
- img = imaging.FlipV(img)
- case RotatedCWMirrored:
- img = imaging.Transpose(img)
- case RotatedCCW:
- img = imaging.Rotate270(img)
- case RotatedCCWMirrored:
- img = imaging.Transverse(img)
- case RotatedCW:
- img = imaging.Rotate90(img)
- }
-
- return &img, width, height
-}
-
-func getImageOrientation(imageData []byte) (int, error) {
- if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
- return Upright, err
- } else {
- if tag, err := exifData.Get("Orientation"); err != nil {
- return Upright, err
- } else {
- orientation, err := tag.Int(0)
- if err != nil {
- return Upright, err
- } else {
- return orientation, nil
- }
- }
- }
-}
-
-func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
- thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
- thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
- imgWidth := float64(width)
- imgHeight := float64(height)
-
- var thumbnail image.Image
- if imgHeight < thumbHeight && imgWidth < thumbWidth {
- thumbnail = img
- } else if imgHeight/imgWidth < thumbHeight/thumbWidth {
- thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
- } else {
- thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
- }
-
- buf := new(bytes.Buffer)
- if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
- l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
- return
- }
-
- if err := app.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
- l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
- return
- }
-}
-
-func generatePreviewImage(img image.Image, previewPath string, width int) {
- var preview image.Image
- if width > int(utils.Cfg.FileSettings.PreviewWidth) {
- preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
- } else {
- preview = img
- }
-
- buf := new(bytes.Buffer)
-
- if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
- l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
- return
- }
-
- if err := app.WriteFile(buf.Bytes(), previewPath); err != nil {
- l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
- return
- }
-}
-
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
info, err := getFileInfoForRequest(c, r, true)
if err != nil {
diff --git a/api/import.go b/api/import.go
deleted file mode 100644
index b93a061fe..000000000
--- a/api/import.go
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api
-
-import (
- "bytes"
- "io"
- "regexp"
- "unicode/utf8"
-
- l4g "github.com/alecthomas/log4go"
- "github.com/mattermost/platform/app"
- "github.com/mattermost/platform/model"
- "github.com/mattermost/platform/utils"
-)
-
-//
-// Import functions are sutible for entering posts and users into the database without
-// some of the usual checks. (IsValid is still run)
-//
-
-func ImportPost(post *model.Post) {
- // Workaround for empty messages, which may be the case if they are webhook posts.
- firstIteration := true
- for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
- firstIteration = false
- var remainder string
- if messageRuneCount > model.POST_MESSAGE_MAX_RUNES {
- remainder = string(([]rune(post.Message))[model.POST_MESSAGE_MAX_RUNES:])
- post.Message = truncateRunes(post.Message, model.POST_MESSAGE_MAX_RUNES)
- } else {
- remainder = ""
- }
-
- post.Hashtags, _ = model.ParseHashtags(post.Message)
-
- if result := <-app.Srv.Store.Post().Save(post); result.Err != nil {
- l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message)
- }
-
- for _, fileId := range post.FileIds {
- if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
- l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err)
- }
- }
-
- post.Id = ""
- post.CreateAt++
- post.Message = remainder
- }
-}
-
-func ImportUser(team *model.Team, user *model.User) *model.User {
- user.MakeNonNil()
-
- user.Roles = model.ROLE_SYSTEM_USER.Id
-
- if result := <-app.Srv.Store.User().Save(user); result.Err != nil {
- l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err)
- return nil
- } else {
- ruser := result.Data.(*model.User)
-
- if cresult := <-app.Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
- l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err)
- }
-
- if err := app.JoinUserToTeam(team, user); err != nil {
- l4g.Error(utils.T("api.import.import_user.join_team.error"), err)
- }
-
- return ruser
- }
-}
-
-func ImportChannel(channel *model.Channel) *model.Channel {
- if result := <-app.Srv.Store.Channel().Save(channel); result.Err != nil {
- return nil
- } else {
- sc := result.Data.(*model.Channel)
-
- return sc
- }
-}
-
-func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
- buf := bytes.NewBuffer(nil)
- io.Copy(buf, file)
- data := buf.Bytes()
-
- fileInfo, err := doUploadFile(teamId, channelId, userId, fileName, data)
- if err != nil {
- return nil, err
- }
-
- img, width, height := prepareImage(data)
- if img != nil {
- generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height)
- generatePreviewImage(*img, fileInfo.PreviewPath, width)
- }
-
- return fileInfo, nil
-}
-
-func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
- linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
- post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
-
- post.AddProp("from_webhook", "true")
-
- if _, ok := props["override_username"]; !ok {
- post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME)
- }
-
- if len(props) > 0 {
- for key, val := range props {
- if key == "attachments" {
- if list, success := val.([]interface{}); success {
- // parse attachment links into Markdown format
- for i, aInt := range list {
- attachment := aInt.(map[string]interface{})
- if aText, ok := attachment["text"].(string); ok {
- aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
- attachment["text"] = aText
- list[i] = attachment
- }
- if aText, ok := attachment["pretext"].(string); ok {
- aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
- attachment["pretext"] = aText
- list[i] = attachment
- }
- if fVal, ok := attachment["fields"]; ok {
- if fields, ok := fVal.([]interface{}); ok {
- // parse attachment field links into Markdown format
- for j, fInt := range fields {
- field := fInt.(map[string]interface{})
- if fValue, ok := field["value"].(string); ok {
- fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})")
- field["value"] = fValue
- fields[j] = field
- }
- }
- attachment["fields"] = fields
- list[i] = attachment
- }
- }
- }
- post.AddProp(key, list)
- }
- } else if key != "from_webhook" {
- post.AddProp(key, val)
- }
- }
- }
-
- ImportPost(post)
-}
diff --git a/api/slackimport.go b/api/slackimport.go
deleted file mode 100644
index a7d50b1b9..000000000
--- a/api/slackimport.go
+++ /dev/null
@@ -1,622 +0,0 @@
-// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api
-
-import (
- "archive/zip"
- "bytes"
- "encoding/json"
- "io"
- "mime/multipart"
- "path/filepath"
- "regexp"
- "strconv"
- "strings"
- "unicode/utf8"
-
- l4g "github.com/alecthomas/log4go"
- "github.com/mattermost/platform/app"
- "github.com/mattermost/platform/model"
- "github.com/mattermost/platform/utils"
-)
-
-type SlackChannel struct {
- Id string `json:"id"`
- Name string `json:"name"`
- Members []string `json:"members"`
- Topic map[string]string `json:"topic"`
- Purpose map[string]string `json:"purpose"`
-}
-
-type SlackUser struct {
- Id string `json:"id"`
- Username string `json:"name"`
- Profile map[string]string `json:"profile"`
-}
-
-type SlackFile struct {
- Id string `json:"id"`
- Title string `json:"title"`
-}
-
-type SlackPost struct {
- User string `json:"user"`
- BotId string `json:"bot_id"`
- BotUsername string `json:"username"`
- Text string `json:"text"`
- TimeStamp string `json:"ts"`
- Type string `json:"type"`
- SubType string `json:"subtype"`
- Comment *SlackComment `json:"comment"`
- Upload bool `json:"upload"`
- File *SlackFile `json:"file"`
- Attachments []SlackAttachment `json:"attachments"`
-}
-
-type SlackComment struct {
- User string `json:"user"`
- Comment string `json:"comment"`
-}
-
-type SlackAttachment struct {
- Id int `json:"id"`
- Text string `json:"text"`
- Pretext string `json:"pretext"`
- Fields []map[string]interface{} `json:"fields"`
-}
-
-func truncateRunes(s string, i int) string {
- runes := []rune(s)
- if len(runes) > i {
- return string(runes[:i])
- }
- return s
-}
-
-func SlackConvertTimeStamp(ts string) int64 {
- timeString := strings.SplitN(ts, ".", 2)[0]
-
- timeStamp, err := strconv.ParseInt(timeString, 10, 64)
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn"))
- return 1
- }
- return timeStamp * 1000 // Convert to milliseconds
-}
-
-func SlackConvertChannelName(channelName string) string {
- newName := strings.Trim(channelName, "_-")
- if len(newName) == 1 {
- return "slack-channel-" + newName
- }
-
- return newName
-}
-
-func SlackParseChannels(data io.Reader) ([]SlackChannel, error) {
- decoder := json.NewDecoder(data)
-
- var channels []SlackChannel
- if err := decoder.Decode(&channels); err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error"))
- return channels, err
- }
- return channels, nil
-}
-
-func SlackParseUsers(data io.Reader) ([]SlackUser, error) {
- decoder := json.NewDecoder(data)
-
- var users []SlackUser
- if err := decoder.Decode(&users); err != nil {
- // This actually returns errors that are ignored.
- // In this case it is erroring because of a null that Slack
- // introduced. So we just return the users here.
- return users, err
- }
- return users, nil
-}
-
-func SlackParsePosts(data io.Reader) ([]SlackPost, error) {
- decoder := json.NewDecoder(data)
-
- var posts []SlackPost
- if err := decoder.Decode(&posts); err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error"))
- return posts, err
- }
- return posts, nil
-}
-
-func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
- // Log header
- log.WriteString(utils.T("api.slackimport.slack_add_users.created"))
- log.WriteString("===============\r\n\r\n")
-
- addedUsers := make(map[string]*model.User)
-
- // Need the team
- var team *model.Team
- if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
- log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
- return addedUsers
- } else {
- team = result.Data.(*model.Team)
- }
-
- for _, sUser := range slackusers {
- firstName := ""
- lastName := ""
- if name, ok := sUser.Profile["first_name"]; ok {
- firstName = name
- }
- if name, ok := sUser.Profile["last_name"]; ok {
- lastName = name
- }
-
- email := sUser.Profile["email"]
-
- password := model.NewId()
-
- // Check for email conflict and use existing user if found
- if result := <-app.Srv.Store.User().GetByEmail(email); result.Err == nil {
- existingUser := result.Data.(*model.User)
- addedUsers[sUser.Id] = existingUser
- if err := app.JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
- log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
- } else {
- log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
- }
- continue
- }
-
- newUser := model.User{
- Username: sUser.Username,
- FirstName: firstName,
- LastName: lastName,
- Email: email,
- Password: password,
- }
-
- if mUser := ImportUser(team, &newUser); mUser != nil {
- addedUsers[sUser.Id] = mUser
- log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
- } else {
- log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
- }
- }
-
- return addedUsers
-}
-
-func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
- var team *model.Team
- if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
- log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
- return nil
- } else {
- team = result.Data.(*model.Team)
- }
-
- password := model.NewId()
- username := "slackimportuser_" + model.NewId()
- email := username + "@localhost"
-
- botUser := model.User{
- Username: username,
- FirstName: "",
- LastName: "",
- Email: email,
- Password: password,
- }
-
- if mUser := ImportUser(team, &botUser); mUser != nil {
- log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
- return mUser
- } else {
- log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
- return nil
- }
-}
-
-func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
- for _, sPost := range posts {
- switch {
- case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Text,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- }
- if sPost.Upload {
- if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true {
- newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
- newPost.Message = sPost.File.Title
- }
- }
- ImportPost(&newPost)
- for _, fileId := range newPost.FileIds {
- if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
- l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
- }
- }
-
- case sPost.Type == "message" && sPost.SubType == "file_comment":
- if sPost.Comment == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug"))
- continue
- } else if sPost.Comment.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
- continue
- } else if users[sPost.Comment.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.Comment.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Comment.Comment,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- }
- ImportPost(&newPost)
- case sPost.Type == "message" && sPost.SubType == "bot_message":
- if botUser == nil {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
- continue
- } else if sPost.BotId == "" {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn"))
- continue
- }
-
- props := make(model.StringInterface)
- props["override_username"] = sPost.BotUsername
- if len(sPost.Attachments) > 0 {
- var mAttachments []interface{}
- for _, attachment := range sPost.Attachments {
- mAttachments = append(mAttachments, map[string]interface{}{
- "text": attachment.Text,
- "pretext": attachment.Pretext,
- "fields": attachment.Fields,
- })
- }
- props["attachments"] = mAttachments
- }
-
- post := &model.Post{
- UserId: botUser.Id,
- ChannelId: channel.Id,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- Message: sPost.Text,
- Type: model.POST_SLACK_ATTACHMENT,
- }
-
- ImportIncomingWebhookPost(post, props)
- case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Text,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- Type: model.POST_JOIN_LEAVE,
- }
- ImportPost(&newPost)
- case sPost.Type == "message" && sPost.SubType == "me_message":
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: "*" + sPost.Text + "*",
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- }
- ImportPost(&newPost)
- case sPost.Type == "message" && sPost.SubType == "channel_topic":
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Text,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- Type: model.POST_HEADER_CHANGE,
- }
- ImportPost(&newPost)
- case sPost.Type == "message" && sPost.SubType == "channel_purpose":
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Text,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- Type: model.POST_PURPOSE_CHANGE,
- }
- ImportPost(&newPost)
- case sPost.Type == "message" && sPost.SubType == "channel_name":
- if sPost.User == "" {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
- continue
- } else if users[sPost.User] == nil {
- l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
- continue
- }
- newPost := model.Post{
- UserId: users[sPost.User].Id,
- ChannelId: channel.Id,
- Message: sPost.Text,
- CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
- Type: model.POST_DISPLAYNAME_CHANGE,
- }
- ImportPost(&newPost)
- default:
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType)
- }
- }
-}
-
-func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) {
- if sPost.File != nil {
- if file, ok := uploads[sPost.File.Id]; ok == true {
- openFile, err := file.Open()
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
- return nil, false
- }
- defer openFile.Close()
-
- uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
- return nil, false
- }
-
- return uploadedFile, true
- } else {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id}))
- return nil, false
- }
- } else {
- l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn"))
- return nil, false
- }
-}
-
-func deactivateSlackBotUser(user *model.User) {
- _, err := app.UpdateActive(user, false)
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
- }
-}
-
-func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
- for _, member := range members {
- if user, ok := users[member]; !ok {
- log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
- } else {
- if _, err := app.AddUserToChannel(user, channel); err != nil {
- log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
- }
- }
- }
-}
-
-func SlackSanitiseChannelProperties(channel model.Channel) model.Channel {
- if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
- l4g.Warn("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
- channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES)
- }
-
- if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH {
- l4g.Warn("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
- channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH]
- }
-
- if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
- l4g.Warn("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
- channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES)
- }
-
- if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES {
- l4g.Warn("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
- channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES)
- }
-
- return channel
-}
-
-func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel {
- // Write Header
- log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
- log.WriteString("=================\r\n\r\n")
-
- addedChannels := make(map[string]*model.Channel)
- for _, sChannel := range slackchannels {
- newChannel := model.Channel{
- TeamId: teamId,
- Type: model.CHANNEL_OPEN,
- DisplayName: sChannel.Name,
- Name: SlackConvertChannelName(sChannel.Name),
- Purpose: sChannel.Purpose["value"],
- Header: sChannel.Topic["value"],
- }
- newChannel = SlackSanitiseChannelProperties(newChannel)
- mChannel := ImportChannel(&newChannel)
- if mChannel == nil {
- // Maybe it already exists?
- if result := <-app.Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
- log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
- continue
- } else {
- mChannel = result.Data.(*model.Channel)
- log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
- }
- }
- addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
- log.WriteString(newChannel.DisplayName + "\r\n")
- addedChannels[sChannel.Id] = mChannel
- SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
- }
-
- return addedChannels
-}
-
-func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost {
- var regexes = make(map[string]*regexp.Regexp, len(users))
- for _, user := range users {
- r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username)
- continue
- }
- regexes["@"+user.Username] = r
- }
-
- // Special cases.
- regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
- regexes["@channel"], _ = regexp.Compile("<!channel>")
- regexes["@all"], _ = regexp.Compile("<!everyone>")
-
- for channelName, channelPosts := range posts {
- for postIdx, post := range channelPosts {
- for mention, r := range regexes {
- post.Text = r.ReplaceAllString(post.Text, mention)
- posts[channelName][postIdx] = post
- }
- }
- }
-
- return posts
-}
-
-func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost {
- var regexes = make(map[string]*regexp.Regexp, len(channels))
- for _, channel := range channels {
- r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
- if err != nil {
- l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name)
- continue
- }
- regexes["~"+channel.Name] = r
- }
-
- for channelName, channelPosts := range posts {
- for postIdx, post := range channelPosts {
- for channelReplace, r := range regexes {
- post.Text = r.ReplaceAllString(post.Text, channelReplace)
- posts[channelName][postIdx] = post
- }
- }
- }
-
- return posts
-}
-
-func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost {
- // Convert URLs in Slack's format to Markdown format.
- regex := regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`)
-
- for channelName, channelPosts := range posts {
- for postIdx, post := range channelPosts {
- posts[channelName][postIdx].Text = regex.ReplaceAllString(post.Text, "[$2]($1)")
- }
- }
-
- return posts
-}
-
-func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
- // Create log file
- log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log"))
-
- zipreader, err := zip.NewReader(fileData, fileSize)
- if err != nil || zipreader.File == nil {
- log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error"))
- return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log
- }
-
- var channels []SlackChannel
- var users []SlackUser
- posts := make(map[string][]SlackPost)
- uploads := make(map[string]*zip.File)
- for _, file := range zipreader.File {
- reader, err := file.Open()
- if err != nil {
- log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
- return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log
- }
- if file.Name == "channels.json" {
- channels, _ = SlackParseChannels(reader)
- } else if file.Name == "users.json" {
- users, _ = SlackParseUsers(reader)
- } else {
- spl := strings.Split(file.Name, "/")
- if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
- newposts, _ := SlackParsePosts(reader)
- channel := spl[0]
- if _, ok := posts[channel]; ok == false {
- posts[channel] = newposts
- } else {
- posts[channel] = append(posts[channel], newposts...)
- }
- } else if len(spl) == 3 && spl[0] == "__uploads" {
- uploads[spl[1]] = file
- }
- }
- }
-
- posts = SlackConvertUserMentions(users, posts)
- posts = SlackConvertChannelMentions(channels, posts)
- posts = SlackConvertPostsMarkup(posts)
-
- addedUsers := SlackAddUsers(teamID, users, log)
- botUser := SlackAddBotUser(teamID, log)
-
- SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
-
- if botUser != nil {
- deactivateSlackBotUser(botUser)
- }
-
- app.InvalidateAllCaches()
-
- log.WriteString(utils.T("api.slackimport.slack_import.notes"))
- log.WriteString("=======\r\n\r\n")
-
- log.WriteString(utils.T("api.slackimport.slack_import.note1"))
- log.WriteString(utils.T("api.slackimport.slack_import.note2"))
- log.WriteString(utils.T("api.slackimport.slack_import.note3"))
-
- return nil, log
-}
diff --git a/api/slackimport_test.go b/api/slackimport_test.go
deleted file mode 100644
index efe6e635f..000000000
--- a/api/slackimport_test.go
+++ /dev/null
@@ -1,240 +0,0 @@
-// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api
-
-import (
- "github.com/mattermost/platform/model"
- "os"
- "strings"
- "testing"
-)
-
-func TestSlackConvertTimeStamp(t *testing.T) {
-
- testTimeStamp := "1469785419.000033"
-
- result := SlackConvertTimeStamp(testTimeStamp)
-
- if result != 1469785419000 {
- t.Fatalf("Unexpected timestamp value %v returned.", result)
- }
-}
-
-func TestSlackConvertChannelName(t *testing.T) {
- var testData = []struct {
- input string
- output string
- }{
- {"test-channel", "test-channel"},
- {"_test_channel_", "test_channel"},
- {"__test", "test"},
- {"-t", "slack-channel-t"},
- {"a", "slack-channel-a"},
- }
-
- for _, td := range testData {
- if td.output != SlackConvertChannelName(td.input) {
- t.Fatalf("Did not convert channel name correctly: %v", td.input)
- }
- }
-}
-
-func TestSlackConvertUserMentions(t *testing.T) {
- users := []SlackUser{
- {Id: "U00000A0A", Username: "firstuser"},
- {Id: "U00000B1B", Username: "seconduser"},
- }
-
- posts := map[string][]SlackPost{
- "test-channel": {
- {
- Text: "<!channel>: Hi guys.",
- },
- {
- Text: "Calling <!here|@here>.",
- },
- {
- Text: "Yo <!everyone>.",
- },
- {
- Text: "Regular user test <@U00000B1B|seconduser> and <@U00000A0A>.",
- },
- },
- }
-
- expectedPosts := map[string][]SlackPost{
- "test-channel": {
- {
- Text: "@channel: Hi guys.",
- },
- {
- Text: "Calling @here.",
- },
- {
- Text: "Yo @all.",
- },
- {
- Text: "Regular user test @seconduser and @firstuser.",
- },
- },
- }
-
- convertedPosts := SlackConvertUserMentions(users, posts)
-
- for channelName, channelPosts := range convertedPosts {
- for postIdx, post := range channelPosts {
- if post.Text != expectedPosts[channelName][postIdx].Text {
- t.Fatalf("Converted post text not as expected: %v", post.Text)
- }
- }
- }
-}
-
-func TestSlackConvertChannelMentions(t *testing.T) {
- channels := []SlackChannel{
- {Id: "C000AA00A", Name: "one"},
- {Id: "C000BB11B", Name: "two"},
- }
-
- posts := map[string][]SlackPost{
- "test-channel": {
- {
- Text: "Go to <#C000AA00A>.",
- },
- {
- User: "U00000A0A",
- Text: "Try <#C000BB11B|two> for this.",
- },
- },
- }
-
- expectedPosts := map[string][]SlackPost{
- "test-channel": {
- {
- Text: "Go to ~one.",
- },
- {
- Text: "Try ~two for this.",
- },
- },
- }
-
- convertedPosts := SlackConvertChannelMentions(channels, posts)
-
- for channelName, channelPosts := range convertedPosts {
- for postIdx, post := range channelPosts {
- if post.Text != expectedPosts[channelName][postIdx].Text {
- t.Fatalf("Converted post text not as expected: %v", post.Text)
- }
- }
- }
-
-}
-
-func TestSlackParseChannels(t *testing.T) {
- file, err := os.Open("../tests/slack-import-test-channels.json")
- if err != nil {
- t.Fatalf("Failed to open data file: %v", err)
- }
-
- channels, err := SlackParseChannels(file)
- if err != nil {
- t.Fatalf("Error occurred parsing channels: %v", err)
- }
-
- if len(channels) != 6 {
- t.Fatalf("Unexpected number of channels: %v", len(channels))
- }
-}
-
-func TestSlackParseUsers(t *testing.T) {
- file, err := os.Open("../tests/slack-import-test-users.json")
- if err != nil {
- t.Fatalf("Failed to open data file: %v", err)
- }
-
- users, err := SlackParseUsers(file)
- if err != nil {
- t.Fatalf("Error occurred parsing users: %v", err)
- }
-
- if len(users) != 11 {
- t.Fatalf("Unexpected number of users: %v", len(users))
- }
-}
-
-func TestSlackParsePosts(t *testing.T) {
- file, err := os.Open("../tests/slack-import-test-posts.json")
- if err != nil {
- t.Fatalf("Failed to open data file: %v", err)
- }
-
- posts, err := SlackParsePosts(file)
- if err != nil {
- t.Fatalf("Error occurred parsing posts: %v", err)
- }
-
- if len(posts) != 8 {
- t.Fatalf("Unexpected number of posts: %v", len(posts))
- }
-}
-
-func TestSlackSanitiseChannelProperties(t *testing.T) {
- c1 := model.Channel{
- DisplayName: "display-name",
- Name: "name",
- Purpose: "The channel purpose",
- Header: "The channel header",
- }
-
- c1s := SlackSanitiseChannelProperties(c1)
- if c1.DisplayName != c1s.DisplayName || c1.Name != c1s.Name || c1.Purpose != c1s.Purpose || c1.Header != c1s.Header {
- t.Fatalf("Unexpected alterations to the channel properties.")
- }
-
- c2 := model.Channel{
- DisplayName: strings.Repeat("abcdefghij", 7),
- Name: strings.Repeat("abcdefghij", 7),
- Purpose: strings.Repeat("0123456789", 30),
- Header: strings.Repeat("0123456789", 120),
- }
-
- c2s := SlackSanitiseChannelProperties(c2)
- if c2s.DisplayName != strings.Repeat("abcdefghij", 6)+"abcd" {
- t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.DisplayName)
- }
-
- if c2s.Name != strings.Repeat("abcdefghij", 6)+"abcd" {
- t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Name)
- }
-
- if c2s.Purpose != strings.Repeat("0123456789", 25) {
- t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Purpose)
- }
-
- if c2s.Header != strings.Repeat("0123456789", 102)+"0123" {
- t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Header)
- }
-}
-
-func TestSlackConvertPostsMarkup(t *testing.T) {
- input := make(map[string][]SlackPost)
- input["test"] = []SlackPost{
- {
- Text: "This message contains a link to <https://google.com|Google>.",
- },
- {
- Text: "This message contains a mailto link to <mailto:me@example.com|me@example.com> in it.",
- },
- }
-
- output := SlackConvertPostsMarkup(input)
-
- if output["test"][0].Text != "This message contains a link to [Google](https://google.com)." {
- t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
- }
- if output["test"][1].Text != "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it." {
- t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
- }
-}
diff --git a/api/team.go b/api/team.go
index 574c22de5..096e0a49f 100644
--- a/api/team.go
+++ b/api/team.go
@@ -516,7 +516,7 @@ func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
switch importFrom {
case "slack":
var err *model.AppError
- if err, log = SlackImport(fileData, fileSize, c.TeamId); err != nil {
+ if err, log = app.SlackImport(fileData, fileSize, c.TeamId); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusBadRequest
}