diff options
-rw-r--r-- | .gitignore | 4 | ||||
-rw-r--r-- | Dockerfile | 11 | ||||
-rw-r--r-- | Makefile | 2 | ||||
-rw-r--r-- | api/file.go | 193 | ||||
-rw-r--r-- | api/file_test.go | 201 | ||||
-rw-r--r-- | api/post.go | 33 | ||||
-rw-r--r-- | api/post_test.go | 2 | ||||
-rw-r--r-- | api/user.go | 43 | ||||
-rw-r--r-- | api/user_test.go | 45 | ||||
-rw-r--r-- | config/config.json | 4 | ||||
-rw-r--r-- | config/config_docker.json | 4 | ||||
-rw-r--r-- | model/client.go | 2 | ||||
-rw-r--r-- | model/utils.go | 2 | ||||
-rw-r--r-- | utils/config.go | 22 | ||||
-rw-r--r-- | utils/urlencode.go | 19 | ||||
-rw-r--r-- | web/react/components/view_image.jsx | 2 |
16 files changed, 383 insertions, 206 deletions
diff --git a/.gitignore b/.gitignore index c899ddd8f..b8017c198 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,7 @@ web/sass-files/sass/.sass-cache/ *config.codekit *.sass-cache *styles.css + +# Default local file storage +data/* +api/data/* diff --git a/Dockerfile b/Dockerfile index 5c389c056..e6c04d541 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ WORKDIR /go # # Install SQL -# +# ENV MYSQL_ROOT_PASSWORD=mostest ENV MYSQL_USER=mmuser @@ -60,7 +60,7 @@ RUN echo "deb http://repo.mysql.com/apt/debian/ wheezy mysql-${MYSQL_MAJOR}" > / RUN apt-get update \ && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install mysql-server \ + && apt-get -y install mysql-server \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql @@ -88,12 +88,15 @@ ADD . /go/src/github.com/mattermost/platform ADD ./docker/main.cf /etc/postfix/ RUN go get github.com/tools/godep -RUN cd /go/src/github.com/mattermost/platform; godep restore +RUN cd /go/src/github.com/mattermost/platform; godep restore RUN go install github.com/mattermost/platform -RUN cd /go/src/github.com/mattermost/platform/web/react; npm install +RUN cd /go/src/github.com/mattermost/platform/web/react; npm install RUN chmod +x /go/src/github.com/mattermost/platform/docker/docker-entry.sh ENTRYPOINT /go/src/github.com/mattermost/platform/docker/docker-entry.sh +# Create default storage directory +RUN mkdir /mattermost/ + # Ports EXPOSE 80 @@ -112,6 +112,8 @@ clean: rm -f web/static/js/bundle*.js rm -f web/static/css/styles.css + rm -rf data/* + rm -rf api/data/* rm -rf logs/* diff --git a/api/file.go b/api/file.go index 5d676b9fd..82cee9d1e 100644 --- a/api/file.go +++ b/api/file.go @@ -18,8 +18,10 @@ import ( _ "image/gif" "image/jpeg" "io" + "io/ioutil" "net/http" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -27,7 +29,7 @@ import ( ) func InitFile(r *mux.Router) { - l4g.Debug("Initializing post api routes") + l4g.Debug("Initializing file api routes") sr := r.PathPrefix("/files").Subrouter() sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST") @@ -36,8 +38,8 @@ func InitFile(r *mux.Router) { } func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - m := r.MultipartForm props := m.Value @@ -94,28 +89,25 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) - ext := filepath.Ext(files[i].Filename) + filename := filepath.Base(files[i].Filename) uid := model.NewId() - path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename + path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename - if model.IsFileExtImage(ext) { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options) - imageNameList = append(imageNameList, uid+"/"+files[i].Filename) - imageDataList = append(imageDataList, buf.Bytes()) - } else { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options) + if err := writeFile(buf.Bytes(), path); err != nil { + c.Err = err + return } - if err != nil { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error()) - return + if model.IsFileExtImage(filepath.Ext(files[i].Filename)) { + imageNameList = append(imageNameList, uid+"/"+filename) + imageDataList = append(imageDataList, buf.Bytes()) } - fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) + encName := utils.UrlEncode(filename) + + fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName resStruct.Filenames = append(resStruct.Filenames, fileUrl) } @@ -127,13 +119,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) { go func() { - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" for i, filename := range filenames { @@ -169,11 +154,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch return } - // Upload thumbnail to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) - if err != nil { - l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil { + l4g.Error("Unable to upload thumbnail channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() @@ -188,19 +170,15 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch } buf := new(bytes.Buffer) - err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) - //err = png.Encode(buf, preview) + err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) if err != nil { l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } - // Upload preview to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) - if err != nil { - l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil { + l4g.Error("Unable to upload preview channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() @@ -215,8 +193,8 @@ type ImageGetResult struct { } func getFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -247,13 +225,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - path := "" if len(teamId) == 26 { path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename @@ -262,7 +233,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { } fileData := make(chan []byte) - asyncGetFile(bucket, path, fileData) + asyncGetFile(path, fileData) if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 { if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) { @@ -283,26 +254,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { f := <-fileData if f == nil { - var f2 []byte - tries := 0 - for { - time.Sleep(3000 * time.Millisecond) - tries++ - - asyncGetFile(bucket, path, fileData) - f2 = <-fileData - - if f2 != nil { - w.Header().Set("Cache-Control", "max-age=2592000, public") - w.Header().Set("Content-Length", strconv.Itoa(len(f2))) - w.Write(f2) - return - } else if tries >= 2 { - break - } - } - - c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path) + c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path) c.Err.StatusCode = http.StatusNotFound return } @@ -312,10 +264,11 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { w.Write(f) } -func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) { +func asyncGetFile(path string, fileData chan []byte) { go func() { - data, getErr := bucket.Get(path) + data, getErr := readFile(path) if getErr != nil { + l4g.Error(getErr) fileData <- nil } else { fileData <- data @@ -329,8 +282,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { c.Err.StatusCode = http.StatusForbidden } - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -344,15 +297,14 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { } matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1) - if len(matches) == 0 || len(matches[0]) < 5 { + if len(matches) == 0 || len(matches[0]) < 4 { c.SetInvalidParam("getPublicLink", "filename") return } - getType := matches[0][1] - channelId := matches[0][2] - userId := matches[0][3] - filename = matches[0][4] + channelId := matches[0][1] + userId := matches[0][2] + filename = matches[0][3] cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) @@ -363,7 +315,7 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { data := model.MapToJson(newProps) hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) - url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) + url := fmt.Sprintf("%s/api/v1/files/get/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) if !c.HasPermissionsToChannel(cchan, "getPublicLink") { return @@ -374,3 +326,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { w.Write([]byte(model.MapToJson(rData))) } + +func writeFile(f []byte, path string) *model.AppError { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + ext := filepath.Ext(path) + + var err error + if model.IsFileExtImage(ext) { + options := s3.Options{} + err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options) + + } else { + options := s3.Options{} + err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options) + } + + if err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error()) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil { + return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error()) + } + + if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error()) + } + } else { + return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } + + return nil +} + +func readFile(path string) ([]byte, *model.AppError) { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + // try to get the file from S3 with some basic retry logic + tries := 0 + for { + tries++ + + f, err := bucket.Get(path) + + if f != nil { + return f, nil + } else if tries >= 3 { + return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error()) + } + time.Sleep(3000 * time.Millisecond) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil { + return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error()) + } else { + return f, nil + } + } else { + return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } +} diff --git a/api/file_test.go b/api/file_test.go index 79ee03c77..566fd69d0 100644 --- a/api/file_test.go +++ b/api/file_test.go @@ -38,7 +38,7 @@ func TestUploadFile(t *testing.T) { body := &bytes.Buffer{} writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("files", "test.png") + part, err := writer.CreateFormFile("files", "../test.png") if err != nil { t.Fatal(err) } @@ -68,12 +68,17 @@ func TestUploadFile(t *testing.T) { } resp, appErr := Client.UploadFile("/files/upload", body.Bytes(), writer.FormDataContentType()) - if utils.IsS3Configured() { + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { if appErr != nil { t.Fatal(appErr) } - filenames := resp.Data.(*model.FileUploadResponse).Filenames + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + if strings.Contains(filename, "../") { + t.Fatal("relative path should have been sanitized out") + } + fileId := strings.Split(filename, ".")[0] var auth aws.Auth auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId @@ -82,12 +87,10 @@ func TestUploadFile(t *testing.T) { s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - fileId := strings.Split(filenames[0], ".")[0] - // wait a bit for files to ready time.Sleep(5 * time.Second) - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0]) + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) if err != nil { t.Fatal(err) } @@ -97,13 +100,38 @@ func TestUploadFile(t *testing.T) { t.Fatal(err) } - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png") + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") if err != nil { t.Fatal(err) } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + if strings.Contains(filename, "../") { + t.Fatal("relative path should have been sanitized out") + } + fileId := strings.Split(filename, ".")[0] + + // wait a bit for files to ready + time.Sleep(5 * time.Second) + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } else { if appErr == nil { - t.Fatal("S3 not configured, should have failed") + t.Fatal("S3 and local storage not configured, should have failed") } } } @@ -123,7 +151,7 @@ func TestGetFile(t *testing.T) { channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -169,8 +197,8 @@ func TestGetFile(t *testing.T) { // wait a bit for files to ready time.Sleep(5 * time.Second) - if _, downErr := Client.GetFile(filenames[0], true); downErr != nil { - t.Fatal("file get failed") + if _, downErr := Client.GetFile(filenames[0], false); downErr != nil { + t.Fatal(downErr) } team2 := &model.Team{DisplayName: "Name", Name: "z-z-" + model.NewId() + "a", Email: "test@nowhere.com", Type: model.TEAM_OPEN} @@ -189,35 +217,35 @@ func TestGetFile(t *testing.T) { Client.LoginByEmail(team2.Name, user2.Email, "pwd") - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr != nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr != nil { t.Fatal(downErr) } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), false); downErr == nil { t.Fatal("Should have errored - missing team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", false); downErr == nil { t.Fatal("Should have errored - bad team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", false); downErr == nil { t.Fatal("Should have errored - bad team id") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - missing hash") } - if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - bad hash") } - if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - missing data") } - if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil { + if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil { t.Fatal("Should have errored - bad data") } @@ -225,28 +253,51 @@ func TestGetFile(t *testing.T) { t.Fatal("Should have errored - user not logged in and link not public") } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - fileId := strings.Split(filenames[0], ".")[0] - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0]) - if err != nil { - t.Fatal(err) - } - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") - if err != nil { - t.Fatal(err) - } - - err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png") - if err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") + if err != nil { + t.Fatal(err) + } + } else { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { if _, downErr := Client.GetFile("/files/get/yxebdmbz5pgupx7q6ez88rw11a/n3btzxu9hbnapqk36iwaxkjxhc/junk.jpg", false); downErr.StatusCode != http.StatusNotImplemented { @@ -274,7 +325,7 @@ func TestGetPublicLink(t *testing.T) { channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -350,26 +401,52 @@ func TestGetPublicLink(t *testing.T) { t.Fatal("should have errored, user not member of channel") } - // perform clean-up on s3 - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - fileId := strings.Split(filenames[0], ".")[0] - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + filenames[0]); err != nil { - t.Fatal(err) - } - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_thumb.jpg"); err != nil { - t.Fatal(err) - } - - if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_preview.png"); err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + // perform clean-up on s3 + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename) + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg") + if err != nil { + t.Fatal(err) + } + + err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg") + if err != nil { + t.Fatal(err) + } + } else { + filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/") + filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1] + fileId := strings.Split(filename, ".")[0] + + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + + path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { data := make(map[string]string) diff --git a/api/post.go b/api/post.go index 2d25f7ab0..268a9be20 100644 --- a/api/post.go +++ b/api/post.go @@ -160,6 +160,39 @@ func CreatePost(c *Context, post *model.Post, doUpdateLastViewed bool) (*model.P post.UserId = c.Session.UserId + if len(post.Filenames) > 0 { + doRemove := false + for i := len(post.Filenames) - 1; i >= 0; i-- { + path := post.Filenames[i] + + doRemove = false + if model.UrlRegex.MatchString(path) { + continue + } else if model.PartialUrlRegex.MatchString(path) { + matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1) + if len(matches) == 0 || len(matches[0]) < 4 { + doRemove = true + } + + channelId := matches[0][1] + if channelId != post.ChannelId { + doRemove = true + } + + userId := matches[0][2] + if userId != post.UserId { + doRemove = true + } + } else { + doRemove = true + } + if doRemove { + l4g.Error("Bad filename discarded, filename=%v", path) + post.Filenames = append(post.Filenames[:i], post.Filenames[i+1:]...) + } + } + } + var rpost *model.Post if result := <-Srv.Store.Post().Save(post); result.Err != nil { return nil, result.Err diff --git a/api/post_test.go b/api/post_test.go index 0cccc74d3..19a88f737 100644 --- a/api/post_test.go +++ b/api/post_test.go @@ -37,7 +37,7 @@ func TestCreatePost(t *testing.T) { channel2 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id} channel2 = Client.Must(Client.CreateChannel(channel2)).Data.(*model.Channel) - filenames := []string{"/api/v1/files/get/12345678901234567890123456/12345678901234567890123456/test.png", "/api/v1/files/get/" + channel1.Id + "/" + user1.Id + "/test.png"} + filenames := []string{"/12345678901234567890123456/12345678901234567890123456/12345678901234567890123456/test.png", "/" + channel1.Id + "/" + user1.Id + "/test.png", "www.mattermost.com/fake/url", "junk"} post1 := &model.Post{ChannelId: channel1.Id, Message: "#hashtag a" + model.NewId() + "a", Filenames: filenames} rpost1, err := Client.CreatePost(post1) diff --git a/api/user.go b/api/user.go index 18c5e863a..7035613ea 100644 --- a/api/user.go +++ b/api/user.go @@ -7,8 +7,6 @@ import ( "bytes" l4g "code.google.com/p/log4go" "fmt" - "github.com/goamz/goamz/aws" - "github.com/goamz/goamz/s3" "github.com/gorilla/mux" "github.com/mattermost/platform/model" "github.com/mattermost/platform/store" @@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError buf := new(bytes.Buffer) if imgErr := png.Encode(buf, img); imgErr != nil { - return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error()) + return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error()) } else { return buf.Bytes(), nil } @@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { return } else { var img []byte - var err *model.AppError - if !utils.IsS3Configured() { - img, err = createProfileImage(result.Data.(*model.User).Username, id) - if err != nil { + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var err *model.AppError + if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil { c.Err = err return } } else { - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png" - if data, getErr := bucket.Get(path); getErr != nil { - img, err = createProfileImage(result.Data.(*model.User).Username, id) - if err != nil { + if data, err := readFile(path); err != nil { + + if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil { c.Err = err return } - options := s3.Options{} - if err := bucket.Put(path, img, "image", s3.Private, options); err != nil { - c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error()) + if err := writeFile(img, path); err != nil { + c.Err = err return } @@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { } func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { return } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - m := r.MultipartForm imageArray, ok := m.File["image"] @@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) { path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png" - options := s3.Options{} - if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil { + if err := writeFile(buf.Bytes(), path); err != nil { c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "") return } diff --git a/api/user_test.go b/api/user_test.go index fbd13492b..e236adeaf 100644 --- a/api/user_test.go +++ b/api/user_test.go @@ -356,6 +356,24 @@ func TestUserCreateImage(t *testing.T) { Client.DoGet("/users/"+user.Id+"/image", "", "") + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { + t.Fatal(err) + } + } else { + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } + } + } func TestUserUploadProfileImage(t *testing.T) { @@ -368,7 +386,7 @@ func TestUserUploadProfileImage(t *testing.T) { user = Client.Must(Client.CreateUser(user, "")).Data.(*model.User) store.Must(Srv.Store.User().VerifyEmail(user.Id)) - if utils.IsS3Configured() { + if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -436,15 +454,22 @@ func TestUserUploadProfileImage(t *testing.T) { Client.DoGet("/users/"+user.Id+"/image", "", "") - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - - if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { - t.Fatal(err) + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil { + t.Fatal(err) + } + } else { + path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png" + if err := os.Remove(path); err != nil { + t.Fatal("Couldn't remove file at " + path) + } } } else { body := &bytes.Buffer{} diff --git a/config/config.json b/config/config.json index e6025ef51..085dd6de6 100644 --- a/config/config.json +++ b/config/config.json @@ -19,7 +19,9 @@ "InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6", "PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4", "ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t", - "AnalyticsUrl": "" + "AnalyticsUrl": "", + "UseLocalStorage": true, + "StorageDirectory": "./data/" }, "SqlSettings": { "DriverName": "mysql", diff --git a/config/config_docker.json b/config/config_docker.json index 9be837072..062cdef65 100644 --- a/config/config_docker.json +++ b/config/config_docker.json @@ -19,7 +19,9 @@ "InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6", "PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4", "ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t", - "AnalyticsUrl": "" + "AnalyticsUrl": "", + "UseLocalStorage": true, + "StorageDirectory": "/mattermost/data/" }, "SqlSettings": { "DriverName": "mysql", diff --git a/model/client.go b/model/client.go index c7e17a6db..9a144095a 100644 --- a/model/client.go +++ b/model/client.go @@ -550,7 +550,7 @@ func (c *Client) GetFile(url string, isFullUrl bool) (*Result, *AppError) { if isFullUrl { rq, _ = http.NewRequest("GET", url, nil) } else { - rq, _ = http.NewRequest("GET", c.Url+url, nil) + rq, _ = http.NewRequest("GET", c.Url+"/files/get"+url, nil) } if len(c.AuthToken) > 0 { diff --git a/model/utils.go b/model/utils.go index 38592b984..093a54e38 100644 --- a/model/utils.go +++ b/model/utils.go @@ -319,6 +319,6 @@ func ClearMentionTags(post string) string { } var UrlRegex = regexp.MustCompile(`^((?:[a-z]+:\/\/)?(?:(?:[a-z0-9\-]+\.)+(?:[a-z]{2}|aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|local|internal))(:[0-9]{1,5})?(?:\/[a-z0-9_\-\.~]+)*(\/([a-z0-9_\-\.]*)(?:\?[a-z0-9+_~\-\.%=&]*)?)?(?:#[a-zA-Z0-9!$&'()*+.=-_~:@/?]*)?)(?:\s+|$)$`) -var PartialUrlRegex = regexp.MustCompile(`/api/v1/files/(get|get_image)/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/(([A-Za-z0-9]+/)?.+\.[A-Za-z0-9]{3,})`) +var PartialUrlRegex = regexp.MustCompile(`/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/((?:[A-Za-z0-9]{26})?.+\.[A-Za-z0-9]{3,})`) var SplitRunes = map[rune]bool{',': true, ' ': true, '.': true, '!': true, '?': true, ':': true, ';': true, '\n': true, '<': true, '>': true, '(': true, ')': true, '{': true, '}': true, '[': true, ']': true, '+': true, '/': true, '\\': true} diff --git a/utils/config.go b/utils/config.go index efa4b263a..e8fa9a477 100644 --- a/utils/config.go +++ b/utils/config.go @@ -18,16 +18,18 @@ const ( ) type ServiceSettings struct { - SiteName string - Mode string - AllowTesting bool - UseSSL bool - Port string - Version string - InviteSalt string - PublicLinkSalt string - ResetSalt string - AnalyticsUrl string + SiteName string + Mode string + AllowTesting bool + UseSSL bool + Port string + Version string + InviteSalt string + PublicLinkSalt string + ResetSalt string + AnalyticsUrl string + UseLocalStorage bool + StorageDirectory string } type SqlSettings struct { diff --git a/utils/urlencode.go b/utils/urlencode.go new file mode 100644 index 000000000..63a8f7880 --- /dev/null +++ b/utils/urlencode.go @@ -0,0 +1,19 @@ +// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "net/url" + "strings" +) + +func UrlEncode(str string) string { + strs := strings.Split(str, " ") + + for i, s := range strs { + strs[i] = url.QueryEscape(s) + } + + return strings.Join(strs, "%20") +} diff --git a/web/react/components/view_image.jsx b/web/react/components/view_image.jsx index 4d5d54e7f..ac0ecf299 100644 --- a/web/react/components/view_image.jsx +++ b/web/react/components/view_image.jsx @@ -155,7 +155,7 @@ module.exports = React.createClass({ var imgClass = "hidden"; if (this.state.loaded[id] && this.state.imgId == id) imgClass = ""; - img[info['path']] = <a key={info['path']} className={imgClass} href={this.props.filenames[id]} target="_blank"><img ref="image" src={preview_filename}/></a>; + img[info['path']] = <a key={info['path']} className={imgClass} href={info.path+"."+info.ext} target="_blank"><img ref="image" src={preview_filename}/></a>; } } |