diff options
author | Jonathan <jonfritz@gmail.com> | 2017-11-30 09:07:04 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-11-30 09:07:04 -0500 |
commit | 375c0632fab03e3fb54865e320585888499c076d (patch) | |
tree | ce6cba679337a82370d5a730c428bef21a9964bf /utils | |
parent | d0d9ba4a7e43301697d1c9f495930e0c0179fdc3 (diff) | |
download | chat-375c0632fab03e3fb54865e320585888499c076d.tar.gz chat-375c0632fab03e3fb54865e320585888499c076d.tar.bz2 chat-375c0632fab03e3fb54865e320585888499c076d.zip |
PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)
* Created message export scheduled task
* Added CLI command to immediately kick off an export job
* Added email addresses for users joining and leaving the channel to the export
* Added support for both MySQL and PostgreSQL
* Fixing gofmt error
* Added a new ChannelMemberHistory store and associated tests
* Updating the ChannelMemberHistory channel as users create/join/leave channels
* Added user email to the message export object so it can be included in the actiance export xml
* Don't fail to log a leave event if a corresponding join event wasn't logged
* Adding copyright notices
* Adding message export settings to daily diagnostics report
* Added System Console integration for message export
* Cleaned up TODOs
* Made batch size configurable
* Added export from timestamp to CLI command
* Made ChannelMemberHistory table updates best effort
* Added a context-based timeout option to the message export CLI
* Minor PR updates/improvements
* Removed unnecessary fields from MessageExport object to reduce query overhead
* Removed JSON functions from the message export query in an effort to optimize performance
* Changed the way that channel member history queries and purges work to better account for edge cases
* Fixing a test I missed with the last refactor
* Added file copy functionality to file backend, improved config validation, added default config values
* Fixed file copy tests
* More concise use of the testing libraries
* Fixed context leak error
* Changed default export path to correctly place an 'export' directory under the 'data' directory
* Can't delete records from a read replica
* Fixed copy file tests
* Start job workers when license is applied, if configured to do so
* Suggestions from the PR
* Moar unit tests
* Fixed test imports
Diffstat (limited to 'utils')
-rw-r--r-- | utils/file_backend.go | 1 | ||||
-rw-r--r-- | utils/file_backend_local.go | 7 | ||||
-rw-r--r-- | utils/file_backend_s3.go | 19 | ||||
-rw-r--r-- | utils/file_backend_test.go | 20 | ||||
-rw-r--r-- | utils/license.go | 1 |
5 files changed, 47 insertions, 1 deletions
diff --git a/utils/file_backend.go b/utils/file_backend.go index 3469a63fb..c7a6c5591 100644 --- a/utils/file_backend.go +++ b/utils/file_backend.go @@ -13,6 +13,7 @@ type FileBackend interface { TestConnection() *model.AppError ReadFile(path string) ([]byte, *model.AppError) + CopyFile(oldPath, newPath string) *model.AppError MoveFile(oldPath, newPath string) *model.AppError WriteFile(f []byte, path string) *model.AppError RemoveFile(path string) *model.AppError diff --git a/utils/file_backend_local.go b/utils/file_backend_local.go index b5e67f8f0..1367ccc1e 100644 --- a/utils/file_backend_local.go +++ b/utils/file_backend_local.go @@ -40,6 +40,13 @@ func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { } } +func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError { + if err := CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go index ed88dc70c..5512b64dc 100644 --- a/utils/file_backend_s3.go +++ b/utils/file_backend_s3.go @@ -95,6 +95,23 @@ func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { } } +func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { s3Clnt, err := b.s3New() if err != nil { @@ -107,7 +124,7 @@ func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) diff --git a/utils/file_backend_test.go b/utils/file_backend_test.go index 0989f783c..098f86bbd 100644 --- a/utils/file_backend_test.go +++ b/utils/file_backend_test.go @@ -86,6 +86,26 @@ func (s *FileBackendTestSuite) TestReadWriteFile() { s.EqualValues(readString, "test") } +func (s *FileBackendTestSuite) TestCopyFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + err := s.backend.WriteFile(b, path1) + s.Nil(err) + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + func (s *FileBackendTestSuite) TestMoveFile() { b := []byte("test") path1 := "tests/" + model.NewId() diff --git a/utils/license.go b/utils/license.go index f4775d338..54bad45b5 100644 --- a/utils/license.go +++ b/utils/license.go @@ -228,6 +228,7 @@ func getClientLicense(l *model.License) map[string]string { props["Company"] = l.Customer.Company props["PhoneNumber"] = l.Customer.PhoneNumber props["EmailNotificationContents"] = strconv.FormatBool(*l.Features.EmailNotificationContents) + props["MessageExport"] = strconv.FormatBool(*l.Features.MessageExport) } return props |