From 6990d052d5e95295e729aae28a0d30bfdcb98573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Thu, 11 Jan 2018 16:57:47 +0100 Subject: [XYZ-6] Add sampledata platform command (#8027) * Add fake dependency * [XYZ-6] Add sampledata platform command * Creating EMOJI_NAME_MAX_LENGTH as a constant and using it where needed --- cmd/platform/mattermost.go | 2 +- cmd/platform/sampledata.go | 628 ++++++++++++++++++++++++++++++++++++++++ cmd/platform/sampledata_test.go | 25 ++ 3 files changed, 654 insertions(+), 1 deletion(-) create mode 100644 cmd/platform/sampledata.go create mode 100644 cmd/platform/sampledata_test.go (limited to 'cmd/platform') diff --git a/cmd/platform/mattermost.go b/cmd/platform/mattermost.go index be2ff8164..0a6e3f3a6 100644 --- a/cmd/platform/mattermost.go +++ b/cmd/platform/mattermost.go @@ -36,7 +36,7 @@ func init() { resetCmd.Flags().Bool("confirm", false, "Confirm you really want to delete everything and a DB backup has been performed.") - rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd, messageExportCmd) + rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd, messageExportCmd, sampleDataCmd) } var rootCmd = &cobra.Command{ diff --git a/cmd/platform/sampledata.go b/cmd/platform/sampledata.go new file mode 100644 index 000000000..3e8bf1093 --- /dev/null +++ b/cmd/platform/sampledata.go @@ -0,0 +1,628 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "path" + "sort" + "strings" + "time" + + "github.com/icrowley/fake" + "github.com/mattermost/mattermost-server/app" + "github.com/spf13/cobra" +) + +var sampleDataCmd = &cobra.Command{ + Use: "sampledata", + Short: "Generate sample data", + RunE: sampleDataCmdF, +} + +func sliceIncludes(vs []string, t string) bool { + for _, v := range vs { + if v == t { + return true + } + } + return false +} + +func randomPastTime(seconds int) int64 { + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0)) + return today.Unix() - int64(rand.Intn(seconds*1000)) +} + +func randomEmoji() string { + emojis := []string{"+1", "-1", "heart", "blush"} + return emojis[rand.Intn(len(emojis))] +} + +func randomReaction(users []string, parentCreateAt int64) app.ReactionImportData { + user := users[rand.Intn(len(users))] + emoji := randomEmoji() + date := parentCreateAt + int64(rand.Intn(100000)) + return app.ReactionImportData{ + User: &user, + EmojiName: &emoji, + CreateAt: &date, + } +} + +func randomReply(users []string, parentCreateAt int64) app.ReplyImportData { + user := users[rand.Intn(len(users))] + message := randomMessage(users) + date := parentCreateAt + int64(rand.Intn(100000)) + return app.ReplyImportData{ + User: &user, + Message: &message, + CreateAt: &date, + } +} + +func randomMessage(users []string) string { + var message string + switch rand.Intn(30) { + case 0: + mention := users[rand.Intn(len(users))] + message = "@" + mention + " " + fake.Sentence() + case 1: + switch rand.Intn(2) { + case 0: + mattermostVideos := []string{"Q4MgnxbpZas", "BFo7E9-Kc_E", "LsMLR-BHsKg", "MRmGDhlMhNA", "mUOPxT7VgWc"} + message = "https://www.youtube.com/watch?v=" + mattermostVideos[rand.Intn(len(mattermostVideos))] + case 1: + mattermostTweets := []string{"943119062334353408", "949370809528832005", "948539688171819009", "939122439115681792", "938061722027425797"} + message = "https://twitter.com/mattermosthq/status/" + mattermostTweets[rand.Intn(len(mattermostTweets))] + } + case 2: + message = "" + if rand.Intn(2) == 0 { + message += fake.Sentence() + } + for i := 0; i < rand.Intn(4)+1; i++ { + message += "\n * " + fake.Word() + } + default: + if rand.Intn(2) == 0 { + message = fake.Sentence() + } else { + message = fake.Paragraph() + } + if rand.Intn(3) == 0 { + message += "\n" + fake.Sentence() + } + if rand.Intn(3) == 0 { + message += "\n" + fake.Sentence() + } + if rand.Intn(3) == 0 { + message += "\n" + fake.Sentence() + } + } + return message +} + +func init() { + sampleDataCmd.Flags().Int64P("seed", "s", 1, "Seed used for generating the random data (Different seeds generate different data).") + sampleDataCmd.Flags().IntP("teams", "t", 2, "The number of sample teams.") + sampleDataCmd.Flags().Int("channels-per-team", 10, "The number of sample channels per team.") + sampleDataCmd.Flags().IntP("users", "u", 15, "The number of sample users.") + sampleDataCmd.Flags().Int("team-memberships", 2, "The number of sample team memberships per user.") + sampleDataCmd.Flags().Int("channel-memberships", 5, "The number of sample channel memberships per user in a team.") + sampleDataCmd.Flags().Int("posts-per-channel", 100, "The number of sample post per channel.") + sampleDataCmd.Flags().Int("direct-channels", 30, "The number of sample direct message channels.") + sampleDataCmd.Flags().Int("posts-per-direct-channel", 15, "The number of sample posts per direct message channel.") + sampleDataCmd.Flags().Int("group-channels", 15, "The number of sample group message channels.") + sampleDataCmd.Flags().Int("posts-per-group-channel", 30, "The number of sample posts per group message channel.") + sampleDataCmd.Flags().IntP("workers", "w", 2, "How many workers to run during the import.") + sampleDataCmd.Flags().String("profile-images", "", "Optional. Path to folder with images to randomly pick as user profile image.") + sampleDataCmd.Flags().StringP("bulk", "b", "", "Optional. Path to write a JSONL bulk file instead of loading into the database.") +} + +func sampleDataCmdF(cmd *cobra.Command, args []string) error { + a, err := initDBCommandContextCobra(cmd) + if err != nil { + return err + } + seed, err := cmd.Flags().GetInt64("seed") + if err != nil { + return errors.New("Invalid seed parameter") + } + bulk, err := cmd.Flags().GetString("bulk") + if err != nil { + return errors.New("Invalid bulk parameter") + } + teams, err := cmd.Flags().GetInt("teams") + if err != nil || teams < 0 { + return errors.New("Invalid teams parameter") + } + channelsPerTeam, err := cmd.Flags().GetInt("channels-per-team") + if err != nil || channelsPerTeam < 0 { + return errors.New("Invalid channels-per-team parameter") + } + users, err := cmd.Flags().GetInt("users") + if err != nil || users < 0 { + return errors.New("Invalid users parameter") + } + teamMemberships, err := cmd.Flags().GetInt("team-memberships") + if err != nil || teamMemberships < 0 { + return errors.New("Invalid team-memberships parameter") + } + channelMemberships, err := cmd.Flags().GetInt("channel-memberships") + if err != nil || channelMemberships < 0 { + return errors.New("Invalid channel-memberships parameter") + } + postsPerChannel, err := cmd.Flags().GetInt("posts-per-channel") + if err != nil || postsPerChannel < 0 { + return errors.New("Invalid posts-per-channel parameter") + } + directChannels, err := cmd.Flags().GetInt("direct-channels") + if err != nil || directChannels < 0 { + return errors.New("Invalid direct-channels parameter") + } + postsPerDirectChannel, err := cmd.Flags().GetInt("posts-per-direct-channel") + if err != nil || postsPerDirectChannel < 0 { + return errors.New("Invalid posts-per-direct-channel parameter") + } + groupChannels, err := cmd.Flags().GetInt("group-channels") + if err != nil || groupChannels < 0 { + return errors.New("Invalid group-channels parameter") + } + postsPerGroupChannel, err := cmd.Flags().GetInt("posts-per-group-channel") + if err != nil || postsPerGroupChannel < 0 { + return errors.New("Invalid posts-per-group-channel parameter") + } + workers, err := cmd.Flags().GetInt("workers") + if err != nil { + return errors.New("Invalid workers parameter") + } + profileImagesPath, err := cmd.Flags().GetString("profile-images") + if err != nil { + return errors.New("Invalid profile-images parameter") + } + profileImages := []string{} + if profileImagesPath != "" { + profileImagesStat, err := os.Stat(profileImagesPath) + if os.IsNotExist(err) { + return errors.New("Profile images folder doesn't exists.") + } + if !profileImagesStat.IsDir() { + return errors.New("profile-images parameters must be a folder path.") + } + profileImagesFiles, err := ioutil.ReadDir(profileImagesPath) + if err != nil { + return errors.New("Invalid profile-images parameter") + } + for _, profileImage := range profileImagesFiles { + profileImages = append(profileImages, path.Join(profileImagesPath, profileImage.Name())) + } + sort.Strings(profileImages) + } + + if workers < 1 { + return errors.New("You must have at least one worker.") + } + if teamMemberships > teams { + return errors.New("You can't have more team memberships than teams.") + } + if channelMemberships > channelsPerTeam { + return errors.New("You can't have more channel memberships than channels per team.") + } + + var bulkFile *os.File + switch bulk { + case "": + bulkFile, err = ioutil.TempFile("", ".mattermost-sample-data-") + defer os.Remove(bulkFile.Name()) + if err != nil { + return errors.New("Unable to open temporary file.") + } + case "-": + bulkFile = os.Stdout + default: + bulkFile, err = os.OpenFile(bulk, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return errors.New("Unable to write into the \"" + bulk + "\" file.") + } + } + + encoder := json.NewEncoder(bulkFile) + version := 1 + encoder.Encode(app.LineImportData{Type: "version", Version: &version}) + + fake.Seed(seed) + rand.Seed(seed) + + teamsAndChannels := make(map[string][]string) + for i := 0; i < teams; i++ { + teamLine := createTeam(i) + teamsAndChannels[*teamLine.Team.Name] = []string{} + encoder.Encode(teamLine) + } + + teamsList := []string{} + for teamName := range teamsAndChannels { + teamsList = append(teamsList, teamName) + } + sort.Strings(teamsList) + + for _, teamName := range teamsList { + for i := 0; i < channelsPerTeam; i++ { + channelLine := createChannel(i, teamName) + teamsAndChannels[teamName] = append(teamsAndChannels[teamName], *channelLine.Channel.Name) + encoder.Encode(channelLine) + } + } + + allUsers := []string{} + for i := 0; i < users; i++ { + userLine := createUser(i, teamMemberships, channelMemberships, teamsAndChannels, profileImages) + encoder.Encode(userLine) + allUsers = append(allUsers, *userLine.User.Username) + } + + for team, channels := range teamsAndChannels { + for _, channel := range channels { + for i := 0; i < postsPerChannel; i++ { + postLine := createPost(team, channel, allUsers) + encoder.Encode(postLine) + } + } + } + + for i := 0; i < directChannels; i++ { + user1 := allUsers[rand.Intn(len(allUsers))] + user2 := allUsers[rand.Intn(len(allUsers))] + channelLine := createDirectChannel([]string{user1, user2}) + encoder.Encode(channelLine) + for j := 0; j < postsPerDirectChannel; j++ { + postLine := createDirectPost([]string{user1, user2}) + encoder.Encode(postLine) + } + } + + for i := 0; i < groupChannels; i++ { + users := []string{} + totalUsers := 3 + rand.Intn(3) + for len(users) < totalUsers { + user := allUsers[rand.Intn(len(allUsers))] + if !sliceIncludes(users, user) { + users = append(users, user) + } + } + channelLine := createDirectChannel(users) + encoder.Encode(channelLine) + for j := 0; j < postsPerGroupChannel; j++ { + postLine := createDirectPost(users) + encoder.Encode(postLine) + } + } + + if bulk == "" { + _, err := bulkFile.Seek(0, 0) + if err != nil { + return errors.New("Unable to read correctly the temporary file.") + } + importErr, lineNumber := a.BulkImport(bulkFile, false, workers) + if importErr != nil { + return errors.New(fmt.Sprintf("%s: %s, %s (line: %d)", importErr.Where, importErr.Message, importErr.DetailedError, lineNumber)) + } + } else if bulk != "-" { + err := bulkFile.Close() + if err != nil { + return errors.New("Unable to close correctly the output file") + } + } + + return nil +} + +func createUser(idx int, teamMemberships int, channelMemberships int, teamsAndChannels map[string][]string, profileImages []string) app.LineImportData { + password := fmt.Sprintf("user-%d", idx) + email := fmt.Sprintf("user-%d@sample.mattermost.com", idx) + firstName := fake.FirstName() + lastName := fake.LastName() + username := fmt.Sprintf("%s.%s", strings.ToLower(firstName), strings.ToLower(lastName)) + position := fake.JobTitle() + roles := "system_user" + if idx%5 == 0 { + roles = "system_admin system_user" + } + + // The 75% of the users have custom profile image + var profileImage *string = nil + if rand.Intn(4) != 0 { + profileImageSelector := rand.Int() + if len(profileImages) > 0 { + profileImage = &profileImages[profileImageSelector%len(profileImages)] + } + } + + useMilitaryTime := "false" + if rand.Intn(2) == 0 { + useMilitaryTime = "true" + } + + collapsePreviews := "false" + if rand.Intn(2) == 0 { + collapsePreviews = "true" + } + + messageDisplay := "clean" + if rand.Intn(2) == 0 { + messageDisplay = "compact" + } + + channelDisplayMode := "full" + if rand.Intn(2) == 0 { + channelDisplayMode = "centered" + } + + // Some users has nickname + nickname := "" + if rand.Intn(5) == 0 { + nickname = fake.Company() + } + + // Half of users skip tutorial + tutorialStep := "999" + switch rand.Intn(6) { + case 1: + tutorialStep = "1" + case 2: + tutorialStep = "2" + case 3: + tutorialStep = "3" + } + + teams := []app.UserTeamImportData{} + possibleTeams := []string{} + for teamName := range teamsAndChannels { + possibleTeams = append(possibleTeams, teamName) + } + sort.Strings(possibleTeams) + for x := 0; x < teamMemberships; x++ { + if len(possibleTeams) == 0 { + break + } + position := rand.Intn(len(possibleTeams)) + team := possibleTeams[position] + possibleTeams = append(possibleTeams[:position], possibleTeams[position+1:]...) + if teamChannels, err := teamsAndChannels[team]; err == true { + teams = append(teams, createTeamMembership(channelMemberships, teamChannels, &team)) + } + } + + user := app.UserImportData{ + ProfileImage: profileImage, + Username: &username, + Email: &email, + Password: &password, + Nickname: &nickname, + FirstName: &firstName, + LastName: &lastName, + Position: &position, + Roles: &roles, + Teams: &teams, + UseMilitaryTime: &useMilitaryTime, + CollapsePreviews: &collapsePreviews, + MessageDisplay: &messageDisplay, + ChannelDisplayMode: &channelDisplayMode, + TutorialStep: &tutorialStep, + } + return app.LineImportData{ + Type: "user", + User: &user, + } +} + +func createTeamMembership(numOfchannels int, teamChannels []string, teamName *string) app.UserTeamImportData { + roles := "team_user" + if rand.Intn(5) == 0 { + roles = "team_user team_admin" + } + channels := []app.UserChannelImportData{} + teamChannelsCopy := []string{} + for _, value := range teamChannels { + teamChannelsCopy = append(teamChannelsCopy, value) + } + for x := 0; x < numOfchannels; x++ { + if len(teamChannelsCopy) == 0 { + break + } + position := rand.Intn(len(teamChannelsCopy)) + channelName := teamChannelsCopy[position] + teamChannelsCopy = append(teamChannelsCopy[:position], teamChannelsCopy[position+1:]...) + channels = append(channels, createChannelMembership(channelName)) + } + + return app.UserTeamImportData{ + Name: teamName, + Roles: &roles, + Channels: &channels, + } +} + +func createChannelMembership(channelName string) app.UserChannelImportData { + roles := "channel_user" + if rand.Intn(5) == 0 { + roles = "channel_user channel_admin" + } + favorite := rand.Intn(5) == 0 + + return app.UserChannelImportData{ + Name: &channelName, + Roles: &roles, + Favorite: &favorite, + } +} + +func createTeam(idx int) app.LineImportData { + displayName := fake.Word() + name := fmt.Sprintf("%s-%d", fake.Word(), idx) + allowOpenInvite := rand.Intn(2) == 0 + + description := fake.Paragraph() + if len(description) > 255 { + description = description[0:255] + } + + teamType := "O" + if rand.Intn(2) == 0 { + teamType = "I" + } + + team := app.TeamImportData{ + DisplayName: &displayName, + Name: &name, + AllowOpenInvite: &allowOpenInvite, + Description: &description, + Type: &teamType, + } + return app.LineImportData{ + Type: "team", + Team: &team, + } +} + +func createChannel(idx int, teamName string) app.LineImportData { + displayName := fake.Word() + name := fmt.Sprintf("%s-%d", fake.Word(), idx) + header := fake.Paragraph() + purpose := fake.Paragraph() + + if len(purpose) > 250 { + purpose = purpose[0:250] + } + + channelType := "P" + if rand.Intn(2) == 0 { + channelType = "O" + } + + channel := app.ChannelImportData{ + Team: &teamName, + Name: &name, + DisplayName: &displayName, + Type: &channelType, + Header: &header, + Purpose: &purpose, + } + return app.LineImportData{ + Type: "channel", + Channel: &channel, + } +} + +func createPost(team string, channel string, allUsers []string) app.LineImportData { + message := randomMessage(allUsers) + create_at := randomPastTime(50000) + user := allUsers[rand.Intn(len(allUsers))] + + // Some messages are flagged by an user + flagged_by := []string{} + if rand.Intn(10) == 0 { + flagged_by = append(flagged_by, allUsers[rand.Intn(len(allUsers))]) + } + + reactions := []app.ReactionImportData{} + if rand.Intn(10) == 0 { + for { + reactions = append(reactions, randomReaction(allUsers, create_at)) + if rand.Intn(3) == 0 { + break + } + } + } + + replies := []app.ReplyImportData{} + if rand.Intn(10) == 0 { + for { + replies = append(replies, randomReply(allUsers, create_at)) + if rand.Intn(4) == 0 { + break + } + } + } + + post := app.PostImportData{ + Team: &team, + Channel: &channel, + User: &user, + Message: &message, + CreateAt: &create_at, + FlaggedBy: &flagged_by, + Reactions: &reactions, + Replies: &replies, + } + return app.LineImportData{ + Type: "post", + Post: &post, + } +} + +func createDirectChannel(members []string) app.LineImportData { + header := fake.Sentence() + + channel := app.DirectChannelImportData{ + Members: &members, + Header: &header, + } + return app.LineImportData{ + Type: "direct_channel", + DirectChannel: &channel, + } +} + +func createDirectPost(members []string) app.LineImportData { + message := randomMessage(members) + create_at := randomPastTime(50000) + user := members[rand.Intn(len(members))] + + // Some messages are flagged by an user + flagged_by := []string{} + if rand.Intn(10) == 0 { + flagged_by = append(flagged_by, members[rand.Intn(len(members))]) + } + + reactions := []app.ReactionImportData{} + if rand.Intn(10) == 0 { + for { + reactions = append(reactions, randomReaction(members, create_at)) + if rand.Intn(3) == 0 { + break + } + } + } + + replies := []app.ReplyImportData{} + if rand.Intn(10) == 0 { + for { + replies = append(replies, randomReply(members, create_at)) + if rand.Intn(4) == 0 { + break + } + } + } + + post := app.DirectPostImportData{ + ChannelMembers: &members, + User: &user, + Message: &message, + CreateAt: &create_at, + FlaggedBy: &flagged_by, + Reactions: &reactions, + Replies: &replies, + } + return app.LineImportData{ + Type: "direct_post", + DirectPost: &post, + } +} diff --git a/cmd/platform/sampledata_test.go b/cmd/platform/sampledata_test.go new file mode 100644 index 000000000..de28c0856 --- /dev/null +++ b/cmd/platform/sampledata_test.go @@ -0,0 +1,25 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package main + +import ( + "testing" + + "github.com/mattermost/mattermost-server/api" + "github.com/stretchr/testify/require" +) + +func TestSampledataBadParameters(t *testing.T) { + th := api.Setup().InitBasic() + defer th.TearDown() + + // should fail because you need at least 1 worker + require.Error(t, runCommand(t, "sampledata", "--workers", "0")) + + // should fail because you have more team memberships than teams + require.Error(t, runCommand(t, "sampledata", "--teams", "10", "--teams-memberships", "11")) + + // should fail because you have more channel memberships than channels per team + require.Error(t, runCommand(t, "sampledata", "--channels-per-team", "10", "--channel-memberships", "11")) +} -- cgit v1.2.3-1-g7c22