From 375c0632fab03e3fb54865e320585888499c076d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 30 Nov 2017 09:07:04 -0500 Subject: PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612) * Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports --- i18n/en.json | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) (limited to 'i18n') diff --git a/i18n/en.json b/i18n/en.json index f252ebe08..60b289f88 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1332,6 +1332,10 @@ "id": "api.file.move_file.configured.app_error", "translation": "File storage not configured properly. Please configure for either S3 or local server file storage." }, + { + "id": "api.file.move_file.copy_within_s3.app_error", + "translation": "Unable to copy file within S3." + }, { "id": "api.file.move_file.delete_from_s3.app_error", "translation": "Unable to delete file from S3." @@ -3850,6 +3854,18 @@ "id": "ent.data_retention.generic.license.error", "translation": "License does not support Data Retention." }, + { + "id": "ent.message_export.generic.license.error", + "translation": "License does not support Message Export." + }, + { + "id": "ent.elasticsearch.start.create_bulk_processor_failed.app_error", + "translation": "Failed to create Elasticsearch bulk processor" + }, + { + "id": "ent.elasticsearch.start.start_bulk_processor_failed.app_error", + "translation": "Failed to start Elasticsearch bulk processor" + }, { "id": "ent.elasticsearch.aggregator_worker.create_index_job.error", "translation": "Elasticsearch aggregator worker failed to create the indexing job" @@ -4378,6 +4394,54 @@ "id": "model.channel_member.is_valid.user_id.app_error", "translation": "Invalid user id" }, + { + "id": "model.channel_member_history.is_valid.channel_id.app_error", + "translation": "Invalid channel id" + }, + { + "id": "model.channel_member_history.is_valid.user_id.app_error", + "translation": "Invalid user id" + }, + { + "id": "model.channel_member_history.is_valid.user_email.app_error", + "translation": "Invalid user email" + }, + { + "id": "model.channel_member_history.is_valid.join_time.app_error", + "translation": "Invalid join time" + }, + { + "id": "model.channel_member_history.is_valid.leave_time.app_error", + "translation": "Invalid leave time" + }, + { + "id": "store.sql_channel_member_history.log_join_event.app_error", + "translation": "Failed to record channel member history" + }, + { + "id": "store.sql_channel_member_history.log_leave_event.select_error", + "translation": "Failed to record channel member history. No existing join record found" + }, + { + "id": "store.sql_channel_member_history.log_leave_event.update_error", + "translation": "Failed to record channel member history. Failed to update existing join record" + }, + { + "id": "store.sql_channel_member_history.get_users_in_channel_at.app_error", + "translation": "Failed to get users in channel at specified time" + }, + { + "id": "store.sql_channel_member_history.get_users_in_channel_during.app_error", + "translation": "Failed to get users in channel during specified time period" + }, + { + "id": "store.sql_channel_member_history.get_all.app_error", + "translation": "Failed to get records" + }, + { + "id": "store.sql_channel_member_history.purge_history_before.app_error", + "translation": "Failed to purge records" + }, { "id": "model.client.connecting.app_error", "translation": "We encountered an error while connecting to the server" @@ -4878,6 +4942,30 @@ "id": "model.config.is_valid.write_timeout.app_error", "translation": "Invalid value for write timeout." }, + { + "id": "model.config.is_valid.message_export.enable.app_error", + "translation": "Message export job EnableExport setting must be either true or false" + }, + { + "id": "model.config.is_valid.message_export.daily_runtime.app_error", + "translation": "Message export job DailyRuntime must be a 24-hour time stamp in the form HH:MM." + }, + { + "id": "model.config.is_valid.message_export.export_from.app_error", + "translation": "Message export job ExportFromTimestamp must be a timestamp (expressed in seconds since unix epoch). Only messages sent after this timestamp will be exported." + }, + { + "id": "model.config.is_valid.message_export.file_location.app_error", + "translation": "Message export job FileLocation must be a writable directory that export data will be written to" + }, + { + "id": "model.config.is_valid.message_export.file_location.relative", + "translation": "Message export job FileLocation must be a sub-directory of FileSettings.Directory" + }, + { + "id": "model.config.is_valid.message_export.batch_size.app_error", + "translation": "Message export job BatchSize must be a positive integer" + }, { "id": "model.emoji.create_at.app_error", "translation": "Create at must be a valid time" @@ -5474,6 +5562,10 @@ "id": "store.sql.upgraded.warn", "translation": "The database schema has been upgraded to version %v" }, + { + "id": "store.sql_compliance.message_export.app_error", + "translation": "Failed to select message export data" + }, { "id": "store.sql_audit.get.finding.app_error", "translation": "We encountered an error finding the audits" -- cgit v1.2.3-1-g7c22