From 0e2b321e6f5ab5983bc3428aa455dac7012c36ee Mon Sep 17 00:00:00 2001 From: Joram Wilander Date: Fri, 13 Jan 2017 15:17:50 -0500 Subject: Refactor and migrate more functions out of api into app package (#5063) --- app/file.go | 340 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 app/file.go (limited to 'app/file.go') diff --git a/app/file.go b/app/file.go new file mode 100644 index 000000000..93a286a14 --- /dev/null +++ b/app/file.go @@ -0,0 +1,340 @@ +// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "sync" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" + + s3 "github.com/minio/minio-go" +) + +func ReadFile(path string) ([]byte, *model.AppError) { + if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket + minioObject, err := s3Clnt.GetObject(bucket, path) + defer minioObject.Close() + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } else { + return f, nil + } + } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error()) + } else { + return f, nil + } + } else { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "") + } +} + +func MoveFile(oldPath, newPath string) *model.AppError { + if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket + + var copyConds = s3.NewCopyConditions() + if err = s3Clnt.CopyObject(bucket, newPath, "/"+path.Join(bucket, oldPath), copyConds); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) + } + if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) + } + } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) + } + + if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) + } + } else { + return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "") + } + + return nil +} + +func WriteFile(f []byte, path string) *model.AppError { + if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket + ext := filepath.Ext(path) + + if model.IsFileExtImage(ext) { + _, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), model.GetImageMimeType(ext)) + } else { + _, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), "binary/octet-stream") + } + if err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil { + return err + } + } else { + return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "") + } + + return nil +} + +func writeFileLocally(f []byte, path string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error()) + } + + if err := ioutil.WriteFile(path, f, 0644); err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error()) + } + + return nil +} + +func openFileWriteStream(path string) (io.Writer, *model.AppError) { + if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.s3.app_error", nil, "") + } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+path), 0774); err != nil { + return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.creating_dir.app_error", nil, err.Error()) + } + + if fileHandle, err := os.Create(utils.Cfg.FileSettings.Directory + path); err != nil { + return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.local_server.app_error", nil, err.Error()) + } else { + fileHandle.Chmod(0644) + return fileHandle, nil + } + } + + return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "") +} + +func closeFileWriteStream(file io.Writer) { + file.(*os.File).Close() +} + +func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { + // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} + split := strings.SplitN(filename, "/", 5) + if len(split) < 5 { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.unexpected_filename.error"), post.Id, filename) + return nil + } + + channelId := split[1] + userId := split[2] + oldId := split[3] + name, _ := url.QueryUnescape(split[4]) + + if split[0] != "" || split[1] != post.ChannelId || split[2] != post.UserId || strings.Contains(split[4], "/") { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.mismatched_filename.warn"), post.Id, post.ChannelId, post.UserId, filename) + } + + pathPrefix := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/", teamId, channelId, userId, oldId) + path := pathPrefix + name + + // Open the file and populate the fields of the FileInfo + var info *model.FileInfo + if data, err := ReadFile(path); err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err) + return nil + } else { + var err *model.AppError + info, err = model.GetInfoForBytes(name, data) + if err != nil { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.info.app_error"), post.Id, filename, err) + } + } + + // Generate a new ID because with the old system, you could very rarely get multiple posts referencing the same file + info.Id = model.NewId() + info.CreatorId = post.UserId + info.PostId = post.Id + info.CreateAt = post.CreateAt + info.UpdateAt = post.UpdateAt + info.Path = path + + if info.IsImage() { + nameWithoutExtension := name[:strings.LastIndex(name, ".")] + info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" + info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" + } + + return info +} + +func FindTeamIdForFilename(post *model.Post, filename string) string { + split := strings.SplitN(filename, "/", 5) + id := split[3] + name, _ := url.QueryUnescape(split[4]) + + // This post is in a direct channel so we need to figure out what team the files are stored under. + if result := <-Srv.Store.Team().GetTeamsByUserId(post.UserId); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.teams.app_error"), post.Id, result.Err) + } else if teams := result.Data.([]*model.Team); len(teams) == 1 { + // The user has only one team so the post must've been sent from it + return teams[0].Id + } else { + for _, team := range teams { + path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) + if _, err := ReadFile(path); err == nil { + // Found the team that this file was posted from + return team.Id + } + } + } + + return "" +} + +var fileMigrationLock sync.Mutex + +// Creates and stores FileInfos for a post created before the FileInfos table existed. +func MigrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo { + if len(post.Filenames) == 0 { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.no_filenames.warn"), post.Id) + return []*model.FileInfo{} + } + + cchan := Srv.Store.Channel().Get(post.ChannelId, true) + + // There's a weird bug that rarely happens where a post ends up with duplicate Filenames so remove those + filenames := utils.RemoveDuplicatesFromStringArray(post.Filenames) + + var channel *model.Channel + if result := <-cchan; result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.channel.app_error"), post.Id, post.ChannelId, result.Err) + return []*model.FileInfo{} + } else { + channel = result.Data.(*model.Channel) + } + + // Find the team that was used to make this post since its part of the file path that isn't saved in the Filename + var teamId string + if channel.TeamId == "" { + // This post was made in a cross-team DM channel so we need to find where its files were saved + teamId = FindTeamIdForFilename(post, filenames[0]) + } else { + teamId = channel.TeamId + } + + // Create FileInfo objects for this post + infos := make([]*model.FileInfo, 0, len(filenames)) + if teamId == "" { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.team_id.error"), post.Id, filenames) + } else { + for _, filename := range filenames { + info := GetInfoForFilename(post, teamId, filename) + if info == nil { + continue + } + + infos = append(infos, info) + } + } + + // Lock to prevent only one migration thread from trying to update the post at once, preventing duplicate FileInfos from being created + fileMigrationLock.Lock() + defer fileMigrationLock.Unlock() + + if result := <-Srv.Store.Post().Get(post.Id); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.get_post_again.app_error"), post.Id, result.Err) + return []*model.FileInfo{} + } else if newPost := result.Data.(*model.PostList).Posts[post.Id]; len(newPost.Filenames) != len(post.Filenames) { + // Another thread has already created FileInfos for this post, so just return those + if result := <-Srv.Store.FileInfo().GetForPost(post.Id); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.get_post_file_infos_again.app_error"), post.Id, result.Err) + return []*model.FileInfo{} + } else { + l4g.Debug(utils.T("api.file.migrate_filenames_to_file_infos.not_migrating_post.debug"), post.Id) + return result.Data.([]*model.FileInfo) + } + } + + l4g.Debug(utils.T("api.file.migrate_filenames_to_file_infos.migrating_post.debug"), post.Id) + + savedInfos := make([]*model.FileInfo, 0, len(infos)) + fileIds := make([]string, 0, len(filenames)) + for _, info := range infos { + if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_file_info.app_error"), post.Id, info.Id, info.Path, result.Err) + continue + } + + savedInfos = append(savedInfos, info) + fileIds = append(fileIds, info.Id) + } + + // Copy and save the updated post + newPost := &model.Post{} + *newPost = *post + + newPost.Filenames = []string{} + newPost.FileIds = fileIds + + // Update Posts to clear Filenames and set FileIds + if result := <-Srv.Store.Post().Update(newPost, post); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_post.app_error"), post.Id, newPost.FileIds, post.Filenames, result.Err) + return []*model.FileInfo{} + } else { + return savedInfos + } +} + +func GeneratePublicLink(siteURL string, info *model.FileInfo) string { + hash := GeneratePublicLinkHash(info.Id, *utils.Cfg.FileSettings.PublicLinkSalt) + return fmt.Sprintf("%s%s/public/files/%v/get?h=%s", siteURL, model.API_URL_SUFFIX, info.Id, hash) +} + +func GeneratePublicLinkHash(fileId, salt string) string { + hash := sha256.New() + hash.Write([]byte(salt)) + hash.Write([]byte(fileId)) + + return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) +} -- cgit v1.2.3-1-g7c22