From 375c0632fab03e3fb54865e320585888499c076d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 30 Nov 2017 09:07:04 -0500 Subject: PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612) * Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports --- store/storetest/mocks/ChannelMemberHistoryStore.go | 77 ++++++++++++++++++++++ store/storetest/mocks/ComplianceStore.go | 16 +++++ store/storetest/mocks/LayeredStoreDatabaseLayer.go | 16 +++++ store/storetest/mocks/Store.go | 16 +++++ 4 files changed, 125 insertions(+) create mode 100644 store/storetest/mocks/ChannelMemberHistoryStore.go (limited to 'store/storetest/mocks') diff --git a/store/storetest/mocks/ChannelMemberHistoryStore.go b/store/storetest/mocks/ChannelMemberHistoryStore.go new file mode 100644 index 000000000..4ac0967f9 --- /dev/null +++ b/store/storetest/mocks/ChannelMemberHistoryStore.go @@ -0,0 +1,77 @@ +// Code generated by mockery v1.0.0 + +// Regenerate this file using `make store-mocks`. + +package mocks + +import mock "github.com/stretchr/testify/mock" +import store "github.com/mattermost/mattermost-server/store" + +// ChannelMemberHistoryStore is an autogenerated mock type for the ChannelMemberHistoryStore type +type ChannelMemberHistoryStore struct { + mock.Mock +} + +// GetUsersInChannelDuring provides a mock function with given fields: startTime, endTime, channelId +func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelId string) store.StoreChannel { + ret := _m.Called(startTime, endTime, channelId) + + var r0 store.StoreChannel + if rf, ok := ret.Get(0).(func(int64, int64, string) store.StoreChannel); ok { + r0 = rf(startTime, endTime, channelId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.StoreChannel) + } + } + + return r0 +} + +// LogJoinEvent provides a mock function with given fields: userId, channelId, joinTime +func (_m *ChannelMemberHistoryStore) LogJoinEvent(userId string, channelId string, joinTime int64) store.StoreChannel { + ret := _m.Called(userId, channelId, joinTime) + + var r0 store.StoreChannel + if rf, ok := ret.Get(0).(func(string, string, int64) store.StoreChannel); ok { + r0 = rf(userId, channelId, joinTime) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.StoreChannel) + } + } + + return r0 +} + +// LogLeaveEvent provides a mock function with given fields: userId, channelId, leaveTime +func (_m *ChannelMemberHistoryStore) LogLeaveEvent(userId string, channelId string, leaveTime int64) store.StoreChannel { + ret := _m.Called(userId, channelId, leaveTime) + + var r0 store.StoreChannel + if rf, ok := ret.Get(0).(func(string, string, int64) store.StoreChannel); ok { + r0 = rf(userId, channelId, leaveTime) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.StoreChannel) + } + } + + return r0 +} + +// PurgeHistoryBefore provides a mock function with given fields: time, channelId +func (_m *ChannelMemberHistoryStore) PurgeHistoryBefore(time int64, channelId string) store.StoreChannel { + ret := _m.Called(time, channelId) + + var r0 store.StoreChannel + if rf, ok := ret.Get(0).(func(int64, string) store.StoreChannel); ok { + r0 = rf(time, channelId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.StoreChannel) + } + } + + return r0 +} diff --git a/store/storetest/mocks/ComplianceStore.go b/store/storetest/mocks/ComplianceStore.go index b2208ead7..fb828cd4b 100644 --- a/store/storetest/mocks/ComplianceStore.go +++ b/store/storetest/mocks/ComplianceStore.go @@ -61,6 +61,22 @@ func (_m *ComplianceStore) GetAll(offset int, limit int) store.StoreChannel { return r0 } +// MessageExport provides a mock function with given fields: after, limit +func (_m *ComplianceStore) MessageExport(after int64, limit int) store.StoreChannel { + ret := _m.Called(after, limit) + + var r0 store.StoreChannel + if rf, ok := ret.Get(0).(func(int64, int) store.StoreChannel); ok { + r0 = rf(after, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.StoreChannel) + } + } + + return r0 +} + // Save provides a mock function with given fields: compliance func (_m *ComplianceStore) Save(compliance *model.Compliance) store.StoreChannel { ret := _m.Called(compliance) diff --git a/store/storetest/mocks/LayeredStoreDatabaseLayer.go b/store/storetest/mocks/LayeredStoreDatabaseLayer.go index c3b8bbb60..9c66c4aac 100644 --- a/store/storetest/mocks/LayeredStoreDatabaseLayer.go +++ b/store/storetest/mocks/LayeredStoreDatabaseLayer.go @@ -46,6 +46,22 @@ func (_m *LayeredStoreDatabaseLayer) Channel() store.ChannelStore { return r0 } +// ChannelMemberHistory provides a mock function with given fields: +func (_m *LayeredStoreDatabaseLayer) ChannelMemberHistory() store.ChannelMemberHistoryStore { + ret := _m.Called() + + var r0 store.ChannelMemberHistoryStore + if rf, ok := ret.Get(0).(func() store.ChannelMemberHistoryStore); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.ChannelMemberHistoryStore) + } + } + + return r0 +} + // Close provides a mock function with given fields: func (_m *LayeredStoreDatabaseLayer) Close() { _m.Called() diff --git a/store/storetest/mocks/Store.go b/store/storetest/mocks/Store.go index 85ed10d35..40b50a554 100644 --- a/store/storetest/mocks/Store.go +++ b/store/storetest/mocks/Store.go @@ -44,6 +44,22 @@ func (_m *Store) Channel() store.ChannelStore { return r0 } +// ChannelMemberHistory provides a mock function with given fields: +func (_m *Store) ChannelMemberHistory() store.ChannelMemberHistoryStore { + ret := _m.Called() + + var r0 store.ChannelMemberHistoryStore + if rf, ok := ret.Get(0).(func() store.ChannelMemberHistoryStore); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(store.ChannelMemberHistoryStore) + } + } + + return r0 +} + // Close provides a mock function with given fields: func (_m *Store) Close() { _m.Called() -- cgit v1.2.3-1-g7c22