diff options
Diffstat (limited to 'api')
-rw-r--r-- | api/file.go | 193 | ||||
-rw-r--r-- | api/file_test.go | 201 | ||||
-rw-r--r-- | api/post.go | 37 | ||||
-rw-r--r-- | api/post_test.go | 2 | ||||
-rw-r--r-- | api/templates/post_subject.html | 2 | ||||
-rw-r--r-- | api/user.go | 43 | ||||
-rw-r--r-- | api/user_test.go | 45 |
7 files changed, 332 insertions, 191 deletions
diff --git a/api/file.go b/api/file.go index 362cdf896..82cee9d1e 100644 --- a/api/file.go +++ b/api/file.go @@ -18,8 +18,10 @@ import ( _ "image/gif" "image/jpeg" "io" + "io/ioutil" "net/http" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -27,7 +29,7 @@ import ( ) func InitFile(r *mux.Router) { - l4g.Debug("Initializing post api routes") + l4g.Debug("Initializing file api routes") sr := r.PathPrefix("/files").Subrouter() sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST") @@ -36,8 +38,8 @@ func InitFile(r *mux.Router) { } func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - m := r.MultipartForm props := m.Value @@ -94,28 +89,25 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) - ext := filepath.Ext(files[i].Filename) + filename := filepath.Base(files[i].Filename) uid := model.NewId() - path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename + path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename - if model.IsFileExtImage(ext) { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options) - imageNameList = append(imageNameList, uid+"/"+files[i].Filename) - imageDataList = append(imageDataList, buf.Bytes()) - } else { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options) + if err := writeFile(buf.Bytes(), path); err != nil { + c.Err = err + return } - if err != nil { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error()) - return + if model.IsFileExtImage(filepath.Ext(files[i].Filename)) { + imageNameList = append(imageNameList, uid+"/"+filename) + imageDataList = append(imageDataList, buf.Bytes()) } - fileUrl := c.GetSiteURL() + "/api/v1/files/get/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) + encName := utils.UrlEncode(filename) + + fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName resStruct.Filenames = append(resStruct.Filenames, fileUrl) } @@ -127,13 +119,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) { go func() { - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" for i, filename := range filenames { @@ -169,11 +154,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch return } - // Upload thumbnail to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) - if err != nil { - l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil { + l4g.Error("Unable to upload thumbnail channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() @@ -188,19 +170,15 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch } buf := new(bytes.Buffer) - err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) - //err = png.Encode(buf, preview) + err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) if err != nil { l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } - // Upload preview to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) - if err != nil { - l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil { + l4g.Error("Unable to upload preview channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() @@ -215,8 +193,8 @@ type ImageGetResult struct { } func getFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -247,13 +225,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - path := "" if len(teamId) == 26 { path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename @@ -262,7 +233,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { } fileData := make(chan []byte) - asyncGetFile(bucket, path, fileData) + asyncGetFile(path, fileData) if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 { if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) { @@ -283,26 +254,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { f := <-fileData if f == nil { - var f2 []byte - tries := 0 - for { - time.Sleep(3000 * time.Millisecond) - tries++ - - asyncGetFile(bucket, path, fileData) - f2 = <-fileData - - if f2 != nil { - w.Header().Set("Cache-Control", "max-age=2592000, public") - w.Header().Set("Content-Length", strconv.Itoa(len(f2))) - w.Write(f2) - return - } else if tries >= 2 { - break - } - } - - c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path) + c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path) c.Err.StatusCode = http.StatusNotFound return } @@ -312,10 +264,11 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { w.Write(f) } -func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) { +func asyncGetFile(path string, fileData chan []byte) { go func() { - data, getErr := bucket.Get(path) + data, getErr := readFile(path) if getErr != nil { + l4g.Error(getErr) fileData <- nil } else { fileData <- data @@ -329,8 +282,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { c.Err.StatusCode = http.StatusForbidden } - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -344,15 +297,14 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { } matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1) - if len(matches) == 0 || len(matches[0]) < 5 { + if len(matches) == 0 || len(matches[0]) < 4 { c.SetInvalidParam("getPublicLink", "filename") return } - getType := matches[0][1] - channelId := matches[0][2] - userId := matches[0][3] - filename = matches[0][4] + channelId := matches[0][1] + userId := matches[0][2] + filename = matches[0][3] cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) @@ -363,7 +315,7 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { data := model.MapToJson(newProps) hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) - url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) + url := fmt.Sprintf("%s/api/v1/files/get/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) if !c.HasPermissionsToChannel(cchan, "getPublicLink") { return @@ -374,3 +326,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { w.Write([]byte(model.MapToJson(rData))) } + +func writeFile(f []byte, path string) *model.AppError { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + ext := filepath.Ext(path) + + var err error + if model.IsFileExtImage(ext) { + options := s3.Options{} + err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options) + + } else { + options := s3.Options{} + err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options) + } + + if err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error()) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil { + return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error()) + } + + if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error()) + } + } else { + return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } + + return nil +} + +func readFile(path string) ([]byte, *model.AppError) { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + // try to get the file from S3 with some basic retry logic + tries := 0 + for { + tries++ + + f, err := bucket.Get(path) + + if f != nil { + return f, nil + } else if tries >= 3 { + return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error()) + } + time.Sleep(3000 * time.Millisecond) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil { + return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error()) + } else { + return f, nil + } + } else { + return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } +} diff --git a/api/file_test.go b/api/file_test.go index 79ee03c77..566fd69d0 100644 --- a/api/file_test.go +++ b/api/file_test.go @@ -38,7 +38,7 @@ func TestUploadFile(t *testing.T) { body := &bytes.Buffer{} writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("files", "test.png") + part, err := writer.CreateFormFile("files", "../test.png") if err != nil { t.Fatal(err) } @@ -68,12 +68,17 @@ func TestUploadFile(t *testing.T) { } resp, appErr := Client.UploadFile("/files/upload", body.Bytes(), writer.FormDataContentType()) - if utils.IsS3Configured() { + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { if appErr != nil { t.Fatal(appErr) } - filenames := resp.Data.(*model.FileUploadResponse).Filenames + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + if strings.Contains(filename, "../") { + t.Fatal("relative path should have been sanitized out") + } + fileId := strings.Split(filename, ".")[0] var auth aws.Auth auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId @@ -82,12 +87,10 @@ func TestUploadFile(t *testing.T) { s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - fileId := strings.Split(filenames[0], ".")[0] - // wait a bit for files to ready time.Sleep(5 * time.Second) - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0]) + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) if err != nil { t.Fatal(err) } @@ -97,13 +100,38 @@ func TestUploadFile(t *testing.T) { t.Fatal(err) } - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png") + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") if err != nil { t.Fatal(err) } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + if strings.Contains(filename, "../") { + t.Fatal("relative path should have been sanitized out") + } + fileId := strings.Split(filename, ".")[0] + + // wait a bit for files to ready + time.Sleep(5 * time.Second) + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } else { if appErr == nil { - t.Fatal("S3 not configured, should have failed") + t.Fatal("S3 and local storage not configured, should have failed") } } } @@ -123,7 +151,7 @@ func TestGetFile(t *testing.T) { channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -169,8 +197,8 @@ func TestGetFile(t *testing.T) { // wait a bit for files to ready time.Sleep(5 * time.Second) - if _, downErr := Client.GetFile(filenames[0], true); downErr != nil { - t.Fatal("file get failed") + if _, downErr := Client.GetFile(filenames[0], false); downErr != nil { + t.Fatal(downErr) } team2 := &model.Team{DisplayName: "Name", Name: "z-z-" + model.NewId() + "a", Email: "test@nowhere.com", Type: model.TEAM_OPEN} @@ -189,35 +217,35 @@ func TestGetFile(t *testing.T) { Client.LoginByEmail(team2.Name, user2.Email, "pwd") - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr != nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr != nil { t.Fatal(downErr) } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), false); downErr == nil { t.Fatal("Should have errored - missing team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", false); downErr == nil { t.Fatal("Should have errored - bad team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", false); downErr == nil { t.Fatal("Should have errored - bad team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - missing hash") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - bad hash") } - if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - missing data") } - if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - bad data") } @@ -225,28 +253,51 @@ func TestGetFile(t *testing.T) { t.Fatal("Should have errored - user not logged in and link not public") } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - fileId := strings.Split(filenames[0], ".")[0] - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0]) - if err != nil { - t.Fatal(err) - } - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") - if err != nil { - t.Fatal(err) - } - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png") - if err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") + if err != nil { + t.Fatal(err) + } + } else { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { if _, downErr := Client.GetFile("/files/get/yxebdmbz5pgupx7q6ez88rw11a/n3btzxu9hbnapqk36iwaxkjxhc/junk.jpg", false); downErr.StatusCode != http.StatusNotImplemented { @@ -274,7 +325,7 @@ func TestGetPublicLink(t *testing.T) { channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -350,26 +401,52 @@ func TestGetPublicLink(t *testing.T) { t.Fatal("should have errored, user not member of channel") } - // perform clean-up on s3 - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - fileId := strings.Split(filenames[0], ".")[0] - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + filenames[0]); err != nil { - t.Fatal(err) - } - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_thumb.jpg"); err != nil { - t.Fatal(err) - } - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_preview.png"); err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + // perform clean-up on s3 + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") + if err != nil { + t.Fatal(err) + } + } else { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { data := make(map[string]string) diff --git a/api/post.go b/api/post.go index fb9fdd1ef..f96320639 100644 --- a/api/post.go +++ b/api/post.go @@ -11,6 +11,7 @@ import ( "github.com/mattermost/platform/store" "github.com/mattermost/platform/utils" "net/http" + "net/url" "path/filepath" "strconv" "strings" @@ -170,16 +171,16 @@ func CreatePost(c *Context, post *model.Post, doUpdateLastViewed bool) (*model.P continue } else if model.PartialUrlRegex.MatchString(path) { matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1) - if len(matches) == 0 || len(matches[0]) < 5 { + if len(matches) == 0 || len(matches[0]) < 4 { doRemove = true } - channelId := matches[0][2] + channelId := matches[0][1] if channelId != post.ChannelId { doRemove = true } - userId := matches[0][3] + userId := matches[0][2] if userId != post.UserId { doRemove = true } @@ -407,6 +408,36 @@ func fireAndForgetNotifications(post *model.Post, teamId, siteURL string) { bodyPage.Props["PostMessage"] = model.ClearMentionTags(post.Message) bodyPage.Props["TeamLink"] = teamURL + "/channels/" + channel.Name + // attempt to fill in a message body if the post doesn't have any text + if len(strings.TrimSpace(bodyPage.Props["PostMessage"])) == 0 && len(post.Filenames) > 0 { + // extract the filenames from their paths and determine what type of files are attached + filenames := make([]string, len(post.Filenames)) + onlyImages := true + for i, filename := range post.Filenames { + var err error + if filenames[i], err = url.QueryUnescape(filepath.Base(filename)); err != nil { + // this should never error since filepath was escaped using url.QueryEscape + filenames[i] = filepath.Base(filename) + } + + ext := filepath.Ext(filename) + onlyImages = onlyImages && model.IsFileExtImage(ext) + } + filenamesString := strings.Join(filenames, ", ") + + var attachmentPrefix string + if onlyImages { + attachmentPrefix = "Image" + } else { + attachmentPrefix = "File" + } + if len(post.Filenames) > 1 { + attachmentPrefix += "s" + } + + bodyPage.Props["PostMessage"] = fmt.Sprintf("%s: %s sent", attachmentPrefix, filenamesString) + } + if err := utils.SendMail(profileMap[id].Email, subjectPage.Render(), bodyPage.Render()); err != nil { l4g.Error("Failed to send mention email successfully email=%v err=%v", profileMap[id].Email, err) } diff --git a/api/post_test.go b/api/post_test.go index e245366ca..cbba83af6 100644 --- a/api/post_test.go +++ b/api/post_test.go @@ -37,7 +37,7 @@ func TestCreatePost(t *testing.T) { channel2 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel2 = Client.Must(Client.CreateChannel(channel2)).Data.(*model.Channel) - filenames := []string{"/api/v1/files/get/12345678901234567890123456/12345678901234567890123456/test.png", "/api/v1/files/get/" + channel1.Id + "/" + user1.Id + "/test.png", "www.mattermost.com/fake/url", "junk"} + filenames := []string{"/12345678901234567890123456/12345678901234567890123456/12345678901234567890123456/test.png", "/" + channel1.Id + "/" + user1.Id + "/test.png", "www.mattermost.com/fake/url", "junk"} post1 := &model.Post{ChannelId: channel1.Id, Message: "#hashtag a" + model.NewId() + "a", Filenames: filenames} rpost1, err := Client.CreatePost(post1) diff --git a/api/templates/post_subject.html b/api/templates/post_subject.html index 8ebc9550b..7d8941549 100644 --- a/api/templates/post_subject.html +++ b/api/templates/post_subject.html @@ -1 +1 @@ -{{define "post_subject"}}[{{.Props.TeamDisplayName}} {{.SiteName}}] {{.Props.SubjectText}} for {{.Props.Month}} {{.Props.Day}}, {{.Props.Year}}{{end}} +{{define "post_subject"}}[{{.SiteName}}] {{.Props.TeamDisplayName}} Team Notifications for {{.Props.Month}} {{.Props.Day}}, {{.Props.Year}}{{end}} diff --git a/api/user.go b/api/user.go index 18c5e863a..7035613ea 100644 --- a/api/user.go +++ b/api/user.go @@ -7,8 +7,6 @@ import ( "bytes" l4g "code.google.com/p/log4go" "fmt" - "github.com/goamz/goamz/aws" - "github.com/goamz/goamz/s3" "github.com/gorilla/mux" "github.com/mattermost/platform/model" "github.com/mattermost/platform/store" @@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError buf := new(bytes.Buffer) if imgErr := png.Encode(buf, img); imgErr != nil { - return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error()) + return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error()) } else { return buf.Bytes(), nil } @@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { return } else { var img []byte - var err *model.AppError - if !utils.IsS3Configured() { - img, err = createProfileImage(result.Data.(*model.User).Username, id) - if err != nil { + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var err *model.AppError + if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil { c.Err = err return } } else { - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png" - if data, getErr := bucket.Get(path); getErr != nil { - img, err = createProfileImage(result.Data.(*model.User).Username, id) - if err != nil { + if data, err := readFile(path); err != nil { + + if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil { c.Err = err return } - options := s3.Options{} - if err := bucket.Put(path, img, "image", s3.Private, options); err != nil { - c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error()) + if err := writeFile(img, path); err != nil { + c.Err = err return } @@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { } func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { return } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - m := r.MultipartForm imageArray, ok := m.File["image"] @@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png" - options := s3.Options{} - if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil { + if err := writeFile(buf.Bytes(), path); err != nil { c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "") return } diff --git a/api/user_test.go b/api/user_test.go index e7a3f153a..8b95bdf55 100644 --- a/api/user_test.go +++ b/api/user_test.go @@ -363,6 +363,24 @@ func TestUserCreateImage(t *testing.T) { Client.DoGet("/users/"+user.Id+"/image", "", "") + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { + t.Fatal(err) + } + } else { + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + } + } func TestUserUploadProfileImage(t *testing.T) { @@ -375,7 +393,7 @@ func TestUserUploadProfileImage(t *testing.T) { user = Client.Must(Client.CreateUser(user, "")).Data.(*model.User) store.Must(Srv.Store.User().VerifyEmail(user.Id)) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -443,15 +461,22 @@ func TestUserUploadProfileImage(t *testing.T) { Client.DoGet("/users/"+user.Id+"/image", "", "") - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { + t.Fatal(err) + } + } else { + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { body := &bytes.Buffer{} |