From b4930fb8e5c41b54272661426ec0633255089eb3 Mon Sep 17 00:00:00 2001 From: Carlos Tadeu Panato Junior Date: Wed, 25 Oct 2017 18:41:16 +0200 Subject: [PLT-7726] adjustments for the mm-server jenkinsfile (#7688) --- api/apitestlib.go | 3 +- api4/apitestlib.go | 2 +- build/Jenkinsfile | 118 ++++++++++++++++++++++++++++++++++------------------- utils/file_test.go | 15 ++++++- utils/inbucket.go | 24 ++++++++--- 5 files changed, 112 insertions(+), 50 deletions(-) diff --git a/api/apitestlib.go b/api/apitestlib.go index b43bb2886..17e2a6623 100644 --- a/api/apitestlib.go +++ b/api/apitestlib.go @@ -6,6 +6,7 @@ package api import ( "fmt" "net" + "os" "strings" "time" @@ -363,7 +364,7 @@ func (me *TestHelper) LoginSystemAdmin() { } func GenerateTestEmail() string { - if utils.Cfg.EmailSettings.SMTPServer != "dockerhost" { + if utils.Cfg.EmailSettings.SMTPServer != "dockerhost" && os.Getenv("CI_INBUCKET_PORT") == "" { return strings.ToLower("success+" + model.NewId() + "@simulator.amazonses.com") } return strings.ToLower(model.NewId() + "@dockerhost") diff --git a/api4/apitestlib.go b/api4/apitestlib.go index 54b142c4b..25d0ecc60 100644 --- a/api4/apitestlib.go +++ b/api4/apitestlib.go @@ -454,7 +454,7 @@ func (me *TestHelper) LinkUserToTeam(user *model.User, team *model.Team) { } func GenerateTestEmail() string { - if utils.Cfg.EmailSettings.SMTPServer != "dockerhost" { + if utils.Cfg.EmailSettings.SMTPServer != "dockerhost" && os.Getenv("CI_INBUCKET_PORT") == "" { return strings.ToLower("success+" + model.NewId() + "@simulator.amazonses.com") } return strings.ToLower(model.NewId() + "@dockerhost") diff --git a/build/Jenkinsfile b/build/Jenkinsfile index e8667b712..2d6b99748 100644 --- a/build/Jenkinsfile +++ b/build/Jenkinsfile @@ -1,44 +1,64 @@ #!/usr/bin/env groovy -podTemplate(label: 'jenkins-slave', +podTemplate(label: 'jenkins-slave', containers: [ containerTemplate( name: 'mattermost-mysql', image: 'mysql:5.7', alwaysPullImage: false, - resourceRequestCpu: '1000m', - resourceLimitCpu: '2000m', - resourceRequestMemory: '2Gi', - resourceLimitMemory: '4Gi', + resourceRequestCpu: '500m', + resourceLimitCpu: '1000m', + resourceRequestMemory: '1Gi', + resourceLimitMemory: '2Gi', ports: [portMapping(name: 'mysql', containerPort: 3306, hostPort: 3306)], - envVars: [envVar(key: 'MYSQL_ROOT_PASSWORD', value: 'mostest'), envVar(key: 'MYSQL_USER', value: 'mmuser'), envVar(key: 'MYSQL_PASSWORD', value: 'mostest'), envVar(key: 'MYSQL_DATABASE', value: 'mattermost_test')] + envVars: [ + envVar(key: 'MYSQL_ROOT_PASSWORD', value: 'mostest'), + envVar(key: 'MYSQL_USER', value: 'mmuser'), + envVar(key: 'MYSQL_PASSWORD', value: 'mostest'), + envVar(key: 'MYSQL_DATABASE', value: 'mattermost_test') + ] ), containerTemplate( - name: 'golang', - image: 'golang:1.8', - ttyEnabled: true, + name: 'golang', + image: 'golang:1.9.1', + ttyEnabled: true, command: 'cat', alwaysPullImage: false, resourceRequestCpu: '1000m', - resourceLimitCpu: '2000m', + resourceLimitCpu: '1000m', resourceRequestMemory: '2Gi', resourceLimitMemory: '4Gi' ), containerTemplate( - name: 'mattermost-inbucket', - image: 'jhillyerd/inbucket:latest', - ports: [portMapping(name: 'inbucket', hostPost: 10025, containerPort: 10025)], + name: 'mattermost-inbucket', + image: 'jhillyerd/inbucket:latest', + resourceRequestCpu: '250m', + resourceLimitCpu: '250m', + resourceRequestMemory: '256Mi', + resourceLimitMemory: '256Mi' + ), + containerTemplate( + name: 'mattermost-redis', + image: 'redis', + ports: [portMapping(name: 'redis', hostPost: 6379, containerPort: 6379)], resourceRequestCpu: '250m', resourceLimitCpu: '250m', resourceRequestMemory: '256Mi', resourceLimitMemory: '256Mi' ), containerTemplate( - name: 'mattermost-openldap', - image: 'osixia/openldap:1.1.9', + name: 'mattermost-openldap', + image: 'osixia/openldap:1.1.9', alwaysPullImage: true, - ports: [portMapping(name: 'openldap1', hostPost: 389, containerPort: 389), portMapping(name: 'openldap2', hostPost: 636, containerPort: 636)], - envVars: [envVar(key: 'LDAP_TLS_VERIFY_CLIENT', value: 'never'), envVar(key: 'LDAP_ORGANISATION', value: 'Mattermost Test'), envVar(key: 'LDAP_DOMAIN', value: 'mm.test.com'), envVar(key: 'LDAP_ADMIN_PASSWORD', value: 'mostest')], + ports: [ + portMapping(name: 'openldap1', hostPost: 389, containerPort: 389), + portMapping(name: 'openldap2', hostPost: 636, containerPort: 636) + ], + envVars: [ + envVar(key: 'LDAP_TLS_VERIFY_CLIENT', value: 'never'), + envVar(key: 'LDAP_ORGANISATION', value: 'Mattermost Test'), + envVar(key: 'LDAP_DOMAIN', value: 'mm.test.com'), envVar(key: 'LDAP_ADMIN_PASSWORD', value: 'mostest') + ], args: "--loglevel debug", resourceRequestCpu: '250m', resourceLimitCpu: '250m', @@ -46,35 +66,42 @@ podTemplate(label: 'jenkins-slave', resourceLimitMemory: '256Mi' ), containerTemplate( - name: 'mattermost-redis', - image: 'redis', - ports: [portMapping(name: 'redis', hostPost: 6379, containerPort: 6379)], + name: 'mattermost-minio', + image: 'minio/minio:latest', + ttyEnabled: false, + args: 'server /data', + alwaysPullImage: true, resourceRequestCpu: '250m', resourceLimitCpu: '250m', resourceRequestMemory: '256Mi', - resourceLimitMemory: '256Mi' - ),/* + resourceLimitMemory: '256Mi', + envVars: [ + envVar(key: 'MINIO_ACCESS_KEY', value: 'minioaccesskey'), + envVar(key: 'MINIO_SECRET_KEY', value: 'miniosecretkey') + ] + ), containerTemplate( - name: 'mattermost-elasticsearch', - image: 'grundleborg/elasticsearch:latest', + name: 'mattermost-node', + image: 'node', + ttyEnabled: true, + command: 'cat', + alwaysPullImage: false, + resourceRequestCpu: '500m', + resourceLimitCpu: '1000m', + resourceRequestMemory: '1Gi', + resourceLimitMemory: '2Gi' + ), + /* + containerTemplate( + name: 'mattermost-elasticsearch', + image: 'grundleborg/elasticsearch:latest', ports: [portMapping(name: 'elasticsearch', hostPost: 9200, containerPort: 9200)], envVars: [envVar(key: 'http.host', value: '0.0.0.0'), envVar(key: 'transport.host', value: '127.0.0.1'), envVar(key: 'ES_JAVA_OPTS', value: '-Xmx250m -Xmx250m')], resourceRequestCpu: '250m', resourceLimitCpu: '250m', resourceRequestMemory: '256Mi', resourceLimitMemory: '256Mi' - ),*/ - containerTemplate( - name: 'mattermost-node', - image: 'node', - ttyEnabled: true, - command: 'cat', - alwaysPullImage: false, - resourceRequestCpu: '1000m', - resourceLimitCpu: '2000m', - resourceRequestMemory: '2Gi', - resourceLimitMemory: '4Gi' - ) + )*/ ] ) { @@ -107,13 +134,18 @@ podTemplate(label: 'jenkins-slave', // Install build deps sh 'apt-get update && apt-get install zip -y' - + // Modify config to run on jenkins sh 'mv /go/src/github.com/mattermost/mattermost-server/config/default.json /go/src/github.com/mattermost/mattermost-server/config/config.json' sh 'cd /go/src/github.com/mattermost/mattermost-server && sed -i \'s/dockerhost/localhost/g\' config/config.json' sh 'cd /go/src/github.com/mattermost/mattermost-server && sed -i \'s/2500/10025/g\' config/config.json' } + container('mattermost-minio') { + // setting up the minio folder + sh 'mkdir -p /data/mattermost-test' + } + // Setup openldap container /*container('mattermost-openldap') { sh 'echo \\"dn: ou=testusers,dc=mm,dc=test,dc=com\\" >> user1' @@ -128,23 +160,25 @@ podTemplate(label: 'jenkins-slave', } stage('Style Checks') { container('golang') { - sh 'cd /go/src/github.com/mattermost/mattermost-server && make check-style BUILD_NUMBER=$BUILD_NUMBER' + sh 'cd /go/src/github.com/mattermost/mattermost-server && make check-style' } } stage('Build') { container('golang') { - sh 'cd /go/src/github.com/mattermost/mattermost-server && make build BUILD_NUMBER=$BUILD_NUMBER' + sh "cd /go/src/github.com/mattermost/mattermost-server && make build BUILD_NUMBER=${env.BUILD_NUMBER}" } container('mattermost-node') { - sh 'cd mattermost-webapp && curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/`git rev-parse HEAD`/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make build' + sh "cd mattermost-webapp && curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/`git rev-parse HEAD`/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make build" } container('golang') { - sh 'cd /go/src/github.com/mattermost/mattermost-server && make package BUILD_NUMBER=$BUILD_NUMBER' + sh "cd /go/src/github.com/mattermost/mattermost-server && make package BUILD_NUMBER=${env.BUILD_NUMBER}" } } stage('Unit Tests') { container('golang') { - sh 'cd /go/src/github.com/mattermost/mattermost-server && make test-te BUILD_NUMBER=$BUILD_NUMBER TESTFLAGS= TESTFLAGSEE=' + withEnv(['CI_HOST=localhost', 'CI_INBUCKET_PORT=10080', 'CI_MINIO_PORT=9000']) { + sh "cd /go/src/github.com/mattermost/mattermost-server && make test-te BUILD_NUMBER=${env.BUILD_NUMBER} TESTFLAGS= TESTFLAGSEE=" + } } } stage('S3 Publish') { diff --git a/utils/file_test.go b/utils/file_test.go index 0ad02de7e..ed2e8683b 100644 --- a/utils/file_test.go +++ b/utils/file_test.go @@ -4,6 +4,8 @@ package utils import ( + "fmt" + "os" "testing" "github.com/stretchr/testify/suite" @@ -53,6 +55,17 @@ func (s *FileTestSuite) SetupTest() { s.amazonS3SSL = *Cfg.FileSettings.AmazonS3SSL // Set up the state for the tests. + s3Host := os.Getenv("CI_HOST") + if s3Host == "" { + s3Host = "dockerhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9001" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) if s.testDriver == model.IMAGE_DRIVER_LOCAL { *Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_LOCAL } else if s.testDriver == model.IMAGE_DRIVER_S3 { @@ -60,7 +73,7 @@ func (s *FileTestSuite) SetupTest() { Cfg.FileSettings.AmazonS3AccessKeyId = "minioaccesskey" Cfg.FileSettings.AmazonS3SecretAccessKey = "miniosecretkey" Cfg.FileSettings.AmazonS3Bucket = "mattermost-test" - Cfg.FileSettings.AmazonS3Endpoint = "dockerhost:9001" + Cfg.FileSettings.AmazonS3Endpoint = s3Endpoint *Cfg.FileSettings.AmazonS3SSL = false } else { s.T().Fatal("Invalid image driver set for test suite.") diff --git a/utils/inbucket.go b/utils/inbucket.go index 53f6c759d..3aef34d9f 100644 --- a/utils/inbucket.go +++ b/utils/inbucket.go @@ -8,13 +8,13 @@ import ( "fmt" "io" "net/http" + "os" "strings" "time" ) const ( - INBUCKET_HOST = "http://dockerhost:9000" - INBUCKET_API = "/api/v1/mailbox/" + INBUCKET_API = "/api/v1/mailbox/" ) // OutputJSONHeader holds the received Header to test sending emails (inbucket) @@ -49,7 +49,7 @@ func GetMailBox(email string) (results JSONMessageHeaderInbucket, err error) { parsedEmail := ParseEmail(email) - url := fmt.Sprintf("%s%s%s", INBUCKET_HOST, INBUCKET_API, parsedEmail) + url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err @@ -88,7 +88,7 @@ func GetMessageFromMailbox(email, id string) (results JSONMessageInbucket, err e var record JSONMessageInbucket - url := fmt.Sprintf("%s%s%s/%s", INBUCKET_HOST, INBUCKET_API, parsedEmail, id) + url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), INBUCKET_API, parsedEmail, id) req, err := http.NewRequest("GET", url, nil) if err != nil { return record, err @@ -112,7 +112,7 @@ func DeleteMailBox(email string) (err error) { parsedEmail := ParseEmail(email) - url := fmt.Sprintf("%s%s%s", INBUCKET_HOST, INBUCKET_API, parsedEmail) + url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) req, err := http.NewRequest("DELETE", url, nil) if err != nil { return err @@ -146,3 +146,17 @@ func RetryInbucket(attempts int, callback func() error) (err error) { } return fmt.Errorf("After %d attempts, last error: %s", attempts, err) } + +func getInbucketHost() (host string) { + + inbucket_host := os.Getenv("CI_HOST") + if inbucket_host == "" { + inbucket_host = "dockerhost" + } + + inbucket_port := os.Getenv("CI_INBUCKET_PORT") + if inbucket_port == "" { + inbucket_port = "9000" + } + return fmt.Sprintf("http://%s:%s", inbucket_host, inbucket_port) +} -- cgit v1.2.3-1-g7c22