From 0f66b6e72621842467d0e368b95ee58f485d4ace Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 9 Oct 2017 10:16:14 -0700 Subject: store/sqlstore cleanup and postgres tests (#7595) * sqlstore cleanup / postgres tests * remove stopped containers * cmd/platform compile fix * remove test-postgres target from makefile --- Makefile | 16 ---- app/admin.go | 2 +- app/post.go | 4 + app/server.go | 2 +- app/user.go | 6 +- cmd/platform/jobserver.go | 3 +- store/sqlstore/audit_store.go | 3 +- store/sqlstore/channel_store.go | 8 +- store/sqlstore/command_store.go | 9 +- store/sqlstore/file_info_store.go | 2 +- store/sqlstore/oauth_store.go | 5 +- store/sqlstore/post_store.go | 18 ++-- store/sqlstore/preference_store.go | 4 +- store/sqlstore/store.go | 1 + store/sqlstore/store_test.go | 104 ++++++++++++++++++++-- store/sqlstore/supplier.go | 86 +++++++++--------- store/sqlstore/supplier_reactions.go | 4 +- store/sqlstore/team_store.go | 4 +- store/sqlstore/upgrade.go | 4 +- store/sqlstore/user_access_token_store.go | 9 +- store/sqlstore/user_store.go | 16 ++-- store/storetest/docker.go | 141 ++++++++++++++++++++++++++++++ 22 files changed, 333 insertions(+), 118 deletions(-) create mode 100644 store/storetest/docker.go diff --git a/Makefile b/Makefile index 07823082b..4bacfa3b4 100644 --- a/Makefile +++ b/Makefile @@ -343,22 +343,6 @@ ifeq ($(BUILD_ENTERPRISE_READY),true) rm -f config/*.key endif -test-postgres: - @echo Testing Postgres - - @sed -i'' -e 's|"DriverName": "mysql"|"DriverName": "postgres"|g' config/config.json - @sed -i'' -e 's|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|g' config/config.json - - $(GO) test $(GOFLAGS) -run=$(TESTS) -test.v -test.timeout=2000s -covermode=count -coverprofile=cprofile.out -coverpkg=$(ALL_PACKAGES_COMMA) github.com/mattermost/mattermost-server/store/sqlstore || exit 1; \ - if [ -f cprofile.out ]; then \ - tail -n +2 cprofile.out >> cover.out; \ - rm cprofile.out; \ - fi; \ - - @sed -i'' -e 's|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|g' config/config.json - @sed -i'' -e 's|"DriverName": "postgres"|"DriverName": "mysql"|g' config/config.json - @rm config/config.json-e - test-server: test-te test-ee internal-test-web-client: diff --git a/app/admin.go b/app/admin.go index dab7e9759..a3f7ffa2a 100644 --- a/app/admin.go +++ b/app/admin.go @@ -187,7 +187,7 @@ func (a *App) RecycleDatabaseConnection() { oldStore := a.Srv.Store l4g.Warn(utils.T("api.admin.recycle_db_start.warn")) - a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster) + a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster) a.Jobs.Store = a.Srv.Store diff --git a/app/post.go b/app/post.go index 497cab5a6..fe9443177 100644 --- a/app/post.go +++ b/app/post.go @@ -613,6 +613,10 @@ func (a *App) SearchPostsInTeam(terms string, userId string, teamId string, isOr return postList, nil } else { + if !*utils.Cfg.ServiceSettings.EnablePostSearch { + return nil, model.NewAppError("SearchPostsInTeam", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented) + } + channels := []store.StoreChannel{} for _, params := range paramsList { diff --git a/app/server.go b/app/server.go index 5f955dd65..8b09bfef0 100644 --- a/app/server.go +++ b/app/server.go @@ -85,7 +85,7 @@ func (a *App) NewServer() { } func (a *App) InitStores() { - a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster) + a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster) } type VaryBy struct{} diff --git a/app/user.go b/app/user.go index b98583f80..edb4961fc 100644 --- a/app/user.go +++ b/app/user.go @@ -438,7 +438,7 @@ func (a *App) GetUsersPage(page int, perPage int, asAdmin bool) ([]*model.User, } func (a *App) GetUsersEtag() string { - return (<-a.Srv.Store.User().GetEtagForAllProfiles()).Data.(string) + return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForAllProfiles()).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) } func (a *App) GetUsersInTeam(teamId string, offset int, limit int) ([]*model.User, *model.AppError) { @@ -492,11 +492,11 @@ func (a *App) GetUsersNotInTeamPage(teamId string, page int, perPage int, asAdmi } func (a *App) GetUsersInTeamEtag(teamId string) string { - return (<-a.Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string) + return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) } func (a *App) GetUsersNotInTeamEtag(teamId string) string { - return (<-a.Srv.Store.User().GetEtagForProfilesNotInTeam(teamId)).Data.(string) + return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForProfilesNotInTeam(teamId)).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) } func (a *App) GetUsersInChannel(channelId string, offset int, limit int) ([]*model.User, *model.AppError) { diff --git a/cmd/platform/jobserver.go b/cmd/platform/jobserver.go index 4f82a21ee..f265f137b 100644 --- a/cmd/platform/jobserver.go +++ b/cmd/platform/jobserver.go @@ -10,6 +10,7 @@ import ( l4g "github.com/alecthomas/log4go" "github.com/mattermost/mattermost-server/store" "github.com/mattermost/mattermost-server/store/sqlstore" + "github.com/mattermost/mattermost-server/utils" "github.com/spf13/cobra" ) @@ -36,7 +37,7 @@ func jobserverCmdF(cmd *cobra.Command, args []string) { } defer l4g.Close() - a.Jobs.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster) + a.Jobs.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster) defer a.Jobs.Store.Close() a.Jobs.LoadLicense() diff --git a/store/sqlstore/audit_store.go b/store/sqlstore/audit_store.go index 3cc0758cc..eb30058c7 100644 --- a/store/sqlstore/audit_store.go +++ b/store/sqlstore/audit_store.go @@ -8,7 +8,6 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/utils" ) type SqlAuditStore struct { @@ -83,7 +82,7 @@ func (s SqlAuditStore) PermanentDeleteByUser(userId string) store.StoreChannel { func (s SqlAuditStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel { return store.Do(func(result *store.StoreResult) { var query string - if *utils.Cfg.SqlSettings.DriverName == "postgres" { + if s.DriverName() == "postgres" { query = "DELETE from Audits WHERE Id = any (array (SELECT Id FROM Audits WHERE CreateAt < :EndTime LIMIT :Limit))" } else { query = "DELETE from Audits WHERE CreateAt < :EndTime LIMIT :Limit" diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index 1ddc887bd..f9042b0a6 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -1042,7 +1042,7 @@ func (s SqlChannelStore) UpdateLastViewedAt(channelIds []string, userId string) var updateQuery string - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { updateQuery = `UPDATE ChannelMembers SET @@ -1053,7 +1053,7 @@ func (s SqlChannelStore) UpdateLastViewedAt(channelIds []string, userId string) WHERE UserId = :UserId AND (` + updateIdQuery + `)` - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { updateQuery = `UPDATE ChannelMembers SET @@ -1253,7 +1253,7 @@ func (s SqlChannelStore) performSearch(searchQuery string, term string, paramete if term == "" { searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", "", 1) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { splitTerm := strings.Fields(term) for i, t := range strings.Fields(term) { if i == len(splitTerm)-1 { @@ -1267,7 +1267,7 @@ func (s SqlChannelStore) performSearch(searchQuery string, term string, paramete searchClause := fmt.Sprintf("AND (%s) @@ to_tsquery('simple', :Term)", "Name || ' ' || DisplayName") searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { splitTerm := strings.Fields(term) for i, t := range strings.Fields(term) { splitTerm[i] = "+" + t + "*" diff --git a/store/sqlstore/command_store.go b/store/sqlstore/command_store.go index d7c53e7e2..0ab09d61c 100644 --- a/store/sqlstore/command_store.go +++ b/store/sqlstore/command_store.go @@ -92,7 +92,14 @@ func (s SqlCommandStore) GetByTrigger(teamId string, trigger string) store.Store return store.Do(func(result *store.StoreResult) { var command model.Command - if err := s.GetReplica().SelectOne(&command, "SELECT * FROM Commands WHERE TeamId = :TeamId AND `Trigger` = :Trigger AND DeleteAt = 0", map[string]interface{}{"TeamId": teamId, "Trigger": trigger}); err != nil { + var query string + if s.DriverName() == "mysql" { + query = "SELECT * FROM Commands WHERE TeamId = :TeamId AND `Trigger` = :Trigger AND DeleteAt = 0" + } else { + query = "SELECT * FROM Commands WHERE TeamId = :TeamId AND \"trigger\" = :Trigger AND DeleteAt = 0" + } + + if err := s.GetReplica().SelectOne(&command, query, map[string]interface{}{"TeamId": teamId, "Trigger": trigger}); err != nil { result.Err = model.NewAppError("SqlCommandStore.GetByTrigger", "store.sql_command.get_by_trigger.app_error", nil, "teamId="+teamId+", trigger="+trigger+", err="+err.Error(), http.StatusInternalServerError) } diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index 18e3a2a1c..165770962 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -219,7 +219,7 @@ func (fs SqlFileInfoStore) PermanentDelete(fileId string) store.StoreChannel { func (s SqlFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel { return store.Do(func(result *store.StoreResult) { var query string - if *utils.Cfg.SqlSettings.DriverName == "postgres" { + if s.DriverName() == "postgres" { query = "DELETE from FileInfo WHERE Id = any (array (SELECT Id FROM FileInfo WHERE CreateAt < :EndTime LIMIT :Limit))" } else { query = "DELETE from FileInfo WHERE CreateAt < :EndTime LIMIT :Limit" diff --git a/store/sqlstore/oauth_store.go b/store/sqlstore/oauth_store.go index 7644ac5dc..30a44b75f 100644 --- a/store/sqlstore/oauth_store.go +++ b/store/sqlstore/oauth_store.go @@ -10,7 +10,6 @@ import ( "github.com/mattermost/gorp" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/utils" ) type SqlOAuthStore struct { @@ -336,9 +335,9 @@ func (as SqlOAuthStore) deleteOAuthAppSessions(transaction *gorp.Transaction, cl result := store.StoreResult{} query := "" - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if as.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = "DELETE FROM Sessions s USING OAuthAccessData o WHERE o.Token = s.Token AND o.ClientId = :Id" - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if as.DriverName() == model.DATABASE_DRIVER_MYSQL { query = "DELETE s.* FROM Sessions s INNER JOIN OAuthAccessData o ON o.Token = s.Token WHERE o.ClientId = :Id" } diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 53af44ca1..a1b25b5c5 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -713,14 +713,6 @@ var specialSearchChar = []string{ func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) store.StoreChannel { return store.Do(func(result *store.StoreResult) { - if !*utils.Cfg.ServiceSettings.EnablePostSearch { - list := model.NewPostList() - result.Data = list - - result.Err = model.NewAppError("SqlPostStore.Search", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v params=%v", teamId, userId, params.ToJson()), http.StatusNotImplemented) - return - } - queryParams := map[string]interface{}{ "TeamId": teamId, "UserId": userId, @@ -833,7 +825,7 @@ func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchP if terms == "" { // we've already confirmed that we have a channel or user to search for searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", "", 1) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { // Parse text for wildcards if wildcard, err := regexp.Compile("\\*($| )"); err == nil { terms = wildcard.ReplaceAllLiteralString(terms, ":* ") @@ -847,7 +839,7 @@ func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchP searchClause := fmt.Sprintf("AND %s @@ to_tsquery(:Terms)", searchType) searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType) searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1) @@ -912,7 +904,7 @@ func (s SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) store.Sto ORDER BY Name DESC LIMIT 30` - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = `SELECT TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, COUNT(DISTINCT Posts.UserId) AS Value @@ -966,7 +958,7 @@ func (s SqlPostStore) AnalyticsPostCountsByDay(teamId string) store.StoreChannel ORDER BY Name DESC LIMIT 30` - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = `SELECT TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value @@ -1110,7 +1102,7 @@ func (s SqlPostStore) GetPostsBatchForIndexing(startTime int64, limit int) store func (s SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel { return store.Do(func(result *store.StoreResult) { var query string - if *utils.Cfg.SqlSettings.DriverName == "postgres" { + if s.DriverName() == "postgres" { query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < :EndTime LIMIT :Limit))" } else { query = "DELETE from Posts WHERE CreateAt < :EndTime LIMIT :Limit" diff --git a/store/sqlstore/preference_store.go b/store/sqlstore/preference_store.go index ddd1fc268..a97c3aea7 100644 --- a/store/sqlstore/preference_store.go +++ b/store/sqlstore/preference_store.go @@ -101,7 +101,7 @@ func (s SqlPreferenceStore) save(transaction *gorp.Transaction, preference *mode "Value": preference.Value, } - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + if s.DriverName() == model.DATABASE_DRIVER_MYSQL { if _, err := transaction.Exec( `INSERT INTO Preferences @@ -112,7 +112,7 @@ func (s SqlPreferenceStore) save(transaction *gorp.Transaction, preference *mode Value = :Value`, params); err != nil { result.Err = model.NewAppError("SqlPreferenceStore.save", "store.sql_preference.save.updating.app_error", nil, err.Error(), http.StatusInternalServerError) } - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { // postgres has no way to upsert values until version 9.5 and trying inserting and then updating causes transactions to abort count, err := transaction.SelectInt( `SELECT diff --git a/store/sqlstore/store.go b/store/sqlstore/store.go index 02fcaa1cb..93d3bf39f 100644 --- a/store/sqlstore/store.go +++ b/store/sqlstore/store.go @@ -40,6 +40,7 @@ import ( }*/ type SqlStore interface { + DriverName() string GetCurrentSchemaVersion() string GetMaster() *gorp.DbMap GetSearchReplica() *gorp.DbMap diff --git a/store/sqlstore/store_test.go b/store/sqlstore/store_test.go index 605c73b6a..d99c7e441 100644 --- a/store/sqlstore/store_test.go +++ b/store/sqlstore/store_test.go @@ -4,22 +4,110 @@ package sqlstore import ( + "flag" + "os" + "sync" "testing" + "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" + "github.com/mattermost/mattermost-server/store/storetest" "github.com/mattermost/mattermost-server/utils" ) -var sqlStore store.Store +var storeTypes = []*struct { + Name string + Func func() (*storetest.RunningContainer, *model.SqlSettings, error) + Container *storetest.RunningContainer + Store store.Store +}{ + { + Name: "MySQL", + Func: storetest.NewMySQLContainer, + }, + { + Name: "PostgreSQL", + Func: storetest.NewPostgreSQLContainer, + }, +} func StoreTest(t *testing.T, f func(*testing.T, store.Store)) { - if sqlStore == nil { - utils.TranslationsPreInit() - utils.LoadConfig("config.json") - utils.InitTranslations(utils.Cfg.LocalizationSettings) - sqlStore = store.NewLayeredStore(NewSqlSupplier(nil), nil, nil) + defer func() { + if err := recover(); err != nil { + tearDownStores() + panic(err) + } + }() + for _, st := range storeTypes { + st := st + t.Run(st.Name, func(t *testing.T) { f(t, st.Store) }) + } +} - sqlStore.MarkSystemRanUnitTests() +func initStores() { + defer func() { + if err := recover(); err != nil { + tearDownStores() + panic(err) + } + }() + var wg sync.WaitGroup + errCh := make(chan error, len(storeTypes)) + wg.Add(len(storeTypes)) + for _, st := range storeTypes { + st := st + go func() { + defer wg.Done() + container, settings, err := st.Func() + if err != nil { + errCh <- err + return + } + st.Container = container + st.Store = store.NewLayeredStore(NewSqlSupplier(*settings, nil), nil, nil) + st.Store.MarkSystemRanUnitTests() + }() + } + wg.Wait() + select { + case err := <-errCh: + panic(err) + default: } - f(t, sqlStore) +} + +var tearDownStoresOnce sync.Once + +func tearDownStores() { + tearDownStoresOnce.Do(func() { + var wg sync.WaitGroup + wg.Add(len(storeTypes)) + for _, st := range storeTypes { + st := st + go func() { + st.Store.Close() + st.Container.Stop() + wg.Done() + }() + } + wg.Wait() + }) +} + +func TestMain(m *testing.M) { + flag.Parse() + + utils.TranslationsPreInit() + utils.LoadConfig("config.json") + utils.InitTranslations(utils.Cfg.LocalizationSettings) + + status := 0 + + initStores() + defer func() { + tearDownStores() + os.Exit(status) + }() + + status = m.Run() } diff --git a/store/sqlstore/supplier.go b/store/sqlstore/supplier.go index 7d10fef36..f90639f3e 100644 --- a/store/sqlstore/supplier.go +++ b/store/sqlstore/supplier.go @@ -96,12 +96,14 @@ type SqlSupplier struct { replicas []*gorp.DbMap searchReplicas []*gorp.DbMap oldStores SqlSupplierOldStores + settings *model.SqlSettings } -func NewSqlSupplier(metrics einterfaces.MetricsInterface) *SqlSupplier { +func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlSupplier { supplier := &SqlSupplier{ rrCounter: 0, srCounter: 0, + settings: &settings, } supplier.initConnection() @@ -173,8 +175,8 @@ func (s *SqlSupplier) Next() store.LayeredStoreSupplier { return s.next } -func setupConnection(con_type string, driver string, dataSource string, maxIdle int, maxOpen int, trace bool) *gorp.DbMap { - db, err := dbsql.Open(driver, dataSource) +func setupConnection(con_type string, dataSource string, settings *model.SqlSettings) *gorp.DbMap { + db, err := dbsql.Open(*settings.DriverName, dataSource) if err != nil { l4g.Critical(utils.T("store.sql.open_conn.critical"), err) time.Sleep(time.Second) @@ -200,19 +202,19 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle } } - db.SetMaxIdleConns(maxIdle) - db.SetMaxOpenConns(maxOpen) + db.SetMaxIdleConns(*settings.MaxIdleConns) + db.SetMaxOpenConns(*settings.MaxOpenConns) db.SetConnMaxLifetime(time.Duration(MAX_DB_CONN_LIFETIME) * time.Minute) var dbmap *gorp.DbMap - connectionTimeout := time.Duration(*utils.Cfg.SqlSettings.QueryTimeout) * time.Second + connectionTimeout := time.Duration(*settings.QueryTimeout) * time.Second - if driver == "sqlite3" { + if *settings.DriverName == "sqlite3" { dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.SqliteDialect{}, QueryTimeout: connectionTimeout} - } else if driver == model.DATABASE_DRIVER_MYSQL { + } else if *settings.DriverName == model.DATABASE_DRIVER_MYSQL { dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8MB4"}, QueryTimeout: connectionTimeout} - } else if driver == model.DATABASE_DRIVER_POSTGRES { + } else if *settings.DriverName == model.DATABASE_DRIVER_POSTGRES { dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.PostgresDialect{}, QueryTimeout: connectionTimeout} } else { l4g.Critical(utils.T("store.sql.dialect_driver.critical")) @@ -220,7 +222,7 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle os.Exit(EXIT_NO_DRIVER) } - if trace { + if settings.Trace { dbmap.TraceOn("", sqltrace.New(os.Stdout, "sql-trace:", sqltrace.Lmicroseconds)) } @@ -228,34 +230,32 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle } func (s *SqlSupplier) initConnection() { - s.master = setupConnection("master", *utils.Cfg.SqlSettings.DriverName, - *utils.Cfg.SqlSettings.DataSource, *utils.Cfg.SqlSettings.MaxIdleConns, - *utils.Cfg.SqlSettings.MaxOpenConns, utils.Cfg.SqlSettings.Trace) + s.master = setupConnection("master", *s.settings.DataSource, s.settings) - if len(utils.Cfg.SqlSettings.DataSourceReplicas) == 0 { + if len(s.settings.DataSourceReplicas) == 0 { s.replicas = make([]*gorp.DbMap, 1) s.replicas[0] = s.master } else { - s.replicas = make([]*gorp.DbMap, len(utils.Cfg.SqlSettings.DataSourceReplicas)) - for i, replica := range utils.Cfg.SqlSettings.DataSourceReplicas { - s.replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), *utils.Cfg.SqlSettings.DriverName, replica, - *utils.Cfg.SqlSettings.MaxIdleConns, *utils.Cfg.SqlSettings.MaxOpenConns, - utils.Cfg.SqlSettings.Trace) + s.replicas = make([]*gorp.DbMap, len(s.settings.DataSourceReplicas)) + for i, replica := range s.settings.DataSourceReplicas { + s.replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), replica, s.settings) } } - if len(utils.Cfg.SqlSettings.DataSourceSearchReplicas) == 0 { + if len(s.settings.DataSourceSearchReplicas) == 0 { s.searchReplicas = s.replicas } else { - s.searchReplicas = make([]*gorp.DbMap, len(utils.Cfg.SqlSettings.DataSourceSearchReplicas)) - for i, replica := range utils.Cfg.SqlSettings.DataSourceSearchReplicas { - s.searchReplicas[i] = setupConnection(fmt.Sprintf("search-replica-%v", i), *utils.Cfg.SqlSettings.DriverName, replica, - *utils.Cfg.SqlSettings.MaxIdleConns, *utils.Cfg.SqlSettings.MaxOpenConns, - utils.Cfg.SqlSettings.Trace) + s.searchReplicas = make([]*gorp.DbMap, len(s.settings.DataSourceSearchReplicas)) + for i, replica := range s.settings.DataSourceSearchReplicas { + s.searchReplicas[i] = setupConnection(fmt.Sprintf("search-replica-%v", i), replica, s.settings) } } } +func (ss *SqlSupplier) DriverName() string { + return *ss.settings.DriverName +} + func (ss *SqlSupplier) GetCurrentSchemaVersion() string { version, _ := ss.GetMaster().SelectStr("SELECT Value FROM Systems WHERE Name='Version'") return version @@ -281,7 +281,7 @@ func (ss *SqlSupplier) TotalMasterDbConnections() int { func (ss *SqlSupplier) TotalReadDbConnections() int { - if len(utils.Cfg.SqlSettings.DataSourceReplicas) == 0 { + if len(ss.settings.DataSourceReplicas) == 0 { return 0 } @@ -294,7 +294,7 @@ func (ss *SqlSupplier) TotalReadDbConnections() int { } func (ss *SqlSupplier) TotalSearchDbConnections() int { - if len(utils.Cfg.SqlSettings.DataSourceSearchReplicas) == 0 { + if len(ss.settings.DataSourceSearchReplicas) == 0 { return 0 } @@ -318,7 +318,7 @@ func (ss *SqlSupplier) MarkSystemRanUnitTests() { } func (ss *SqlSupplier) DoesTableExist(tableName string) bool { - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { count, err := ss.GetMaster().SelectInt( `SELECT count(relname) FROM pg_class WHERE relname=$1`, strings.ToLower(tableName), @@ -332,7 +332,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool { return count > 0 - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { count, err := ss.GetMaster().SelectInt( `SELECT @@ -363,7 +363,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool { } func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool { - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { count, err := ss.GetMaster().SelectInt( `SELECT COUNT(0) FROM pg_attribute @@ -386,7 +386,7 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool return count > 0 - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { count, err := ss.GetMaster().SelectInt( `SELECT @@ -423,7 +423,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri return false } - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { _, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + postgresColType + " DEFAULT '" + defaultValue + "'") if err != nil { l4g.Critical(utils.T("store.sql.create_column.critical"), err) @@ -433,7 +433,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri return true - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { _, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + mySqlColType + " DEFAULT '" + defaultValue + "'") if err != nil { l4g.Critical(utils.T("store.sql.create_column.critical"), err) @@ -488,9 +488,9 @@ func (ss *SqlSupplier) RenameColumnIfExists(tableName string, oldColumnName stri } var err error - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " CHANGE " + oldColumnName + " " + newColumnName + " " + colType) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " RENAME COLUMN " + oldColumnName + " TO " + newColumnName) } @@ -510,9 +510,9 @@ func (ss *SqlSupplier) GetMaxLengthOfColumnIfExists(tableName string, columnName var result string var err error - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { result, err = ss.GetMaster().SelectStr("SELECT CHARACTER_MAXIMUM_LENGTH FROM information_schema.columns WHERE table_name = '" + tableName + "' AND COLUMN_NAME = '" + columnName + "'") - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { result, err = ss.GetMaster().SelectStr("SELECT character_maximum_length FROM information_schema.columns WHERE table_name = '" + strings.ToLower(tableName) + "' AND column_name = '" + strings.ToLower(columnName) + "'") } @@ -531,9 +531,9 @@ func (ss *SqlSupplier) AlterColumnTypeIfExists(tableName string, columnName stri } var err error - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " MODIFY " + columnName + " " + mySqlColType) - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + } else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType) } @@ -565,7 +565,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string uniqueStr = "UNIQUE " } - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { _, errExists := ss.GetMaster().SelectStr("SELECT $1::regclass", indexName) // It should fail if the index does not exist if errExists == nil { @@ -587,7 +587,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string time.Sleep(time.Second) os.Exit(EXIT_CREATE_INDEX_POSTGRES) } - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName) if err != nil { @@ -622,7 +622,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) bool { - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES { _, err := ss.GetMaster().SelectStr("SELECT $1::regclass", indexName) // It should fail if the index does not exist if err != nil { @@ -637,7 +637,7 @@ func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) b } return true - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL { count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName) if err != nil { diff --git a/store/sqlstore/supplier_reactions.go b/store/sqlstore/supplier_reactions.go index 1732b16ee..052472bf9 100644 --- a/store/sqlstore/supplier_reactions.go +++ b/store/sqlstore/supplier_reactions.go @@ -147,8 +147,8 @@ func (s *SqlSupplier) ReactionPermanentDeleteBatch(ctx context.Context, endTime result := store.NewSupplierResult() var query string - if *utils.Cfg.SqlSettings.DriverName == "postgres" { - query = "DELETE from Reactions WHERE Id = any (array (SELECT Id FROM Reactions WHERE CreateAt < :EndTime LIMIT :Limit))" + if s.DriverName() == "postgres" { + query = "DELETE from Reactions WHERE CreateAt = any (array (SELECT CreateAt FROM Reactions WHERE CreateAt < :EndTime LIMIT :Limit))" } else { query = "DELETE from Reactions WHERE CreateAt < :EndTime LIMIT :Limit" } diff --git a/store/sqlstore/team_store.go b/store/sqlstore/team_store.go index c819ec61a..da63992dd 100644 --- a/store/sqlstore/team_store.go +++ b/store/sqlstore/team_store.go @@ -266,7 +266,7 @@ func (s SqlTeamStore) GetAllTeamListing() store.StoreChannel { return store.Do(func(result *store.StoreResult) { query := "SELECT * FROM Teams WHERE AllowOpenInvite = 1" - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = "SELECT * FROM Teams WHERE AllowOpenInvite = true" } @@ -289,7 +289,7 @@ func (s SqlTeamStore) GetAllTeamPageListing(offset int, limit int) store.StoreCh return store.Do(func(result *store.StoreResult) { query := "SELECT * FROM Teams WHERE AllowOpenInvite = 1 LIMIT :Limit OFFSET :Offset" - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = "SELECT * FROM Teams WHERE AllowOpenInvite = true LIMIT :Limit OFFSET :Offset" } diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index 657cc31e6..6da95d437 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -135,11 +135,11 @@ func UpgradeDatabaseToVersion33(sqlStore SqlStore) { } // increase size of Value column of Preferences table to match the size of the ThemeProps column - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES { if _, err := transaction.Exec("ALTER TABLE Preferences ALTER COLUMN Value TYPE varchar(2000)"); err != nil { themeMigrationFailed(err) } - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if sqlStore.DriverName() == model.DATABASE_DRIVER_MYSQL { if _, err := transaction.Exec("ALTER TABLE Preferences MODIFY Value text"); err != nil { themeMigrationFailed(err) } diff --git a/store/sqlstore/user_access_token_store.go b/store/sqlstore/user_access_token_store.go index 558b01cd6..2535943c7 100644 --- a/store/sqlstore/user_access_token_store.go +++ b/store/sqlstore/user_access_token_store.go @@ -10,7 +10,6 @@ import ( "github.com/mattermost/gorp" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/utils" ) type SqlUserAccessTokenStore struct { @@ -80,9 +79,9 @@ func (s SqlUserAccessTokenStore) deleteSessionsAndTokensById(transaction *gorp.T result := store.StoreResult{} query := "" - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = "DELETE FROM Sessions s USING UserAccessTokens o WHERE o.Token = s.Token AND o.Id = :Id" - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { query = "DELETE s.* FROM Sessions s INNER JOIN UserAccessTokens o ON o.Token = s.Token WHERE o.Id = :Id" } @@ -132,9 +131,9 @@ func (s SqlUserAccessTokenStore) deleteSessionsandTokensByUser(transaction *gorp result := store.StoreResult{} query := "" - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { query = "DELETE FROM Sessions s USING UserAccessTokens o WHERE o.Token = s.Token AND o.UserId = :UserId" - } else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL { + } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { query = "DELETE s.* FROM Sessions s INNER JOIN UserAccessTokens o ON o.Token = s.Token WHERE o.UserId = :UserId" } diff --git a/store/sqlstore/user_store.go b/store/sqlstore/user_store.go index 5d0e1c50d..aba7c56a3 100644 --- a/store/sqlstore/user_store.go +++ b/store/sqlstore/user_store.go @@ -78,7 +78,7 @@ func (us SqlUserStore) CreateIndexesIfNotExists() { us.CreateIndexIfNotExists("idx_users_create_at", "Users", "CreateAt") us.CreateIndexIfNotExists("idx_users_delete_at", "Users", "DeleteAt") - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if us.DriverName() == model.DATABASE_DRIVER_POSTGRES { us.CreateIndexIfNotExists("idx_users_email_lower", "Users", "lower(Email)") us.CreateIndexIfNotExists("idx_users_username_lower", "Users", "lower(Username)") us.CreateIndexIfNotExists("idx_users_nickname_lower", "Users", "lower(Nickname)") @@ -321,9 +321,9 @@ func (s SqlUserStore) GetEtagForAllProfiles() store.StoreChannel { return store.Do(func(result *store.StoreResult) { updateAt, err := s.GetReplica().SelectInt("SELECT UpdateAt FROM Users ORDER BY UpdateAt DESC LIMIT 1") if err != nil { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis()) } else { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt) } }) } @@ -349,9 +349,9 @@ func (s SqlUserStore) GetEtagForProfiles(teamId string) store.StoreChannel { return store.Do(func(result *store.StoreResult) { updateAt, err := s.GetReplica().SelectInt("SELECT UpdateAt FROM Users, TeamMembers WHERE TeamMembers.TeamId = :TeamId AND Users.Id = TeamMembers.UserId ORDER BY UpdateAt DESC LIMIT 1", map[string]interface{}{"TeamId": teamId}) if err != nil { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis()) } else { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt) } }) } @@ -1066,7 +1066,7 @@ func (us SqlUserStore) performSearch(searchQuery string, term string, options ma for i, term := range splitTerms { fields := []string{} for _, field := range splitFields { - if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { + if us.DriverName() == model.DATABASE_DRIVER_POSTGRES { fields = append(fields, fmt.Sprintf("lower(%s) LIKE lower(%s) escape '*' ", field, fmt.Sprintf(":Term%d", i))) } else { fields = append(fields, fmt.Sprintf("%s LIKE %s escape '*' ", field, fmt.Sprintf(":Term%d", i))) @@ -1159,9 +1159,9 @@ func (us SqlUserStore) GetEtagForProfilesNotInTeam(teamId string) store.StoreCha `, map[string]interface{}{"TeamId": teamId}) if err != nil { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis()) } else { - result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress) + result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt) } }) } diff --git a/store/storetest/docker.go b/store/storetest/docker.go new file mode 100644 index 000000000..ef34541e4 --- /dev/null +++ b/store/storetest/docker.go @@ -0,0 +1,141 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package storetest + +import ( + "encoding/json" + "fmt" + "io" + "net" + "os/exec" + "strings" + "time" + + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/mattermost-server/model" +) + +type Container struct { + Id string + NetworkSettings struct { + Ports map[string][]struct { + HostPort string + } + } +} + +type RunningContainer struct { + Container +} + +func (c *RunningContainer) Stop() error { + l4g.Info("removing container: %v", c.Id) + return exec.Command("docker", "rm", "-f", c.Id).Run() +} + +func NewMySQLContainer() (*RunningContainer, *model.SqlSettings, error) { + container, err := runContainer([]string{ + "-e", "MYSQL_ROOT_PASSWORD=mostest", + "-e", "MYSQL_USER=mmuser", + "-e", "MYSQL_PASSWORD=mostest", + "-e", "MYSQL_DATABASE=mattermost_test", + "--tmpfs", "/var/lib/mysql", + "mysql:5.7", + }) + if err != nil { + return nil, nil, err + } + l4g.Info("Waiting for mysql connectivity") + port := container.NetworkSettings.Ports["3306/tcp"][0].HostPort + if err := waitForPort(port); err != nil { + container.Stop() + return nil, nil, err + } + return container, databaseSettings("mysql", "mmuser:mostest@tcp(127.0.0.1:"+port+")/mattermost_test?charset=utf8mb4,utf8"), nil +} + +func NewPostgreSQLContainer() (*RunningContainer, *model.SqlSettings, error) { + container, err := runContainer([]string{ + "-e", "POSTGRES_USER=mmuser", + "-e", "POSTGRES_PASSWORD=mostest", + "--tmpfs", "/var/lib/postgresql/data", + "postgres:9.4", + }) + if err != nil { + return nil, nil, err + } + l4g.Info("Waiting for postgres connectivity") + port := container.NetworkSettings.Ports["5432/tcp"][0].HostPort + if err := waitForPort(port); err != nil { + container.Stop() + return nil, nil, err + } + return container, databaseSettings("postgres", "postgres://mmuser:mostest@127.0.0.1:"+port+"?sslmode=disable"), nil +} + +func databaseSettings(driver, dataSource string) *model.SqlSettings { + settings := &model.SqlSettings{ + DriverName: &driver, + DataSource: &dataSource, + DataSourceReplicas: []string{}, + DataSourceSearchReplicas: []string{}, + MaxIdleConns: new(int), + MaxOpenConns: new(int), + Trace: false, + AtRestEncryptKey: model.NewRandomString(32), + QueryTimeout: new(int), + } + *settings.MaxIdleConns = 10 + *settings.MaxOpenConns = 100 + *settings.QueryTimeout = 10 + return settings +} + +func runContainer(args []string) (*RunningContainer, error) { + name := "mattermost-storetest-" + model.NewId() + dockerArgs := append([]string{"run", "-d", "-P", "--name", name}, args...) + out, err := exec.Command("docker", dockerArgs...).Output() + if err != nil { + return nil, err + } + id := strings.TrimSpace(string(out)) + out, err = exec.Command("docker", "inspect", id).Output() + if err != nil { + exec.Command("docker", "rm", "-f", id).Run() + return nil, err + } + var containers []Container + if err := json.Unmarshal(out, &containers); err != nil { + exec.Command("docker", "rm", "-f", id).Run() + return nil, err + } + l4g.Info("running container: %v", id) + return &RunningContainer{containers[0]}, nil +} + +func waitForPort(port string) error { + for i := 0; i < 120; i++ { + conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, time.Minute) + if err != nil { + return err + } + if err = conn.SetReadDeadline(time.Now().Add(time.Millisecond * 500)); err != nil { + return err + } + _, err = conn.Read(make([]byte, 1)) + conn.Close() + if err == nil { + return nil + } + if e, ok := err.(net.Error); ok && e.Timeout() { + return nil + } + if err != io.EOF { + return err + } + time.Sleep(time.Millisecond * 200) + } + return fmt.Errorf("timeout waiting for port %v", port) +} -- cgit v1.2.3-1-g7c22