From 302ec17beed9128101ef61d69b45d3ee29e16f1e Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 28 Apr 2017 17:54:04 +0100 Subject: Parallelise Bulk Import. (#6267) * Parallelise Bulk Import. * Set worker count through command line flag. --- cmd/platform/import.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cmd/platform') diff --git a/cmd/platform/import.go b/cmd/platform/import.go index 9cee26a52..ea3e42ad2 100644 --- a/cmd/platform/import.go +++ b/cmd/platform/import.go @@ -35,6 +35,7 @@ var bulkImportCmd = &cobra.Command{ func init() { bulkImportCmd.Flags().Bool("apply", false, "Save the import data to the database. Use with caution - this cannot be reverted.") bulkImportCmd.Flags().Bool("validate", false, "Validate the import data without making any changes to the system.") + bulkImportCmd.Flags().Int("workers", 2, "How many workers to run whilst doing the import.") importCmd.AddCommand( bulkImportCmd, @@ -87,6 +88,11 @@ func bulkImportCmdF(cmd *cobra.Command, args []string) error { return errors.New("Validate flag error") } + workers, err := cmd.Flags().GetInt("workers") + if err != nil { + return errors.New("Workers flag error") + } + if len(args) != 1 { return errors.New("Incorrect number of arguments.") } @@ -110,7 +116,7 @@ func bulkImportCmdF(cmd *cobra.Command, args []string) error { CommandPrettyPrintln("") - if err, lineNumber := app.BulkImport(fileReader, !apply); err != nil { + if err, lineNumber := app.BulkImport(fileReader, !apply, workers); err != nil { CommandPrettyPrintln(err.Error()) if lineNumber != 0 { CommandPrettyPrintln(fmt.Sprintf("Error occurred on data file line %v", lineNumber)) -- cgit v1.2.3-1-g7c22