Merge pull request #76 from lukaszbudnik/dev-v4.1

migrator v4.1

Author: lukaszbudnik

README.md

@@ -26,12 +26,17 @@ Further, there is an official docker image available on docker hub. [lukasz/migr
   * [4. Run migrator from official docker image](#4-run-migrator-from-official-docker-image)
   * [5. Play around with migrator](#5-play-around-with-migrator)
 * [Configuration](#configuration)
+  * [migrator.yaml](#migratoryaml)
+  * [Env variables substitution](#env-variables-substitution)
+  * [Source migrations](#source-migrations)
+    * [Local storage](#local-storage)
+    * [AWS S3](#aws-s3)
+  * [Supported databases](#supported-databases)
 * [Customisation and legacy frameworks support](#customisation-and-legacy-frameworks-support)
   * [Custom tenants support](#custom-tenants-support)
   * [Custom schema placeholder](#custom-schema-placeholder)
   * [Synchonising legacy migrations to migrator](#synchonising-legacy-migrations-to-migrator)
   * [Final comments](#final-comments)
-* [Supported databases](#supported-databases)
 * [Performance](#performance)
 * [Change log](#change-log)
 * [Contributing, code style, running unit & integration tests](#contributing-code-style-running-unit--integration-tests)
@@ -41,7 +46,7 @@ Further, there is an official docker image available on docker hub. [lukasz/migr
 
 migrator exposes a simple REST API described below.
 
-# GET /
+## GET /
 
 Migrator returns build information together with supported API versions.
 
@@ -464,7 +469,11 @@ curl -v -X POST -H "Content-Type: application/json" -d '{"name": "new_tenant", "
 
 # Configuration
 
-migrator requires a simple `migrator.yaml` file:
+Let's see how to configure migrator.
+
+## migrator.yaml
+
+migrator configuration file is a simple YAML file. Take a look at a sample `migrator.yaml` configuration file which contains the description, correct syntax, and sample values for all available properties.
 
 ```yaml
 # required, base directory where all migrations are stored, see singleSchemas and tenantSchemas below
@@ -506,6 +515,8 @@ webHookHeaders:
   - "X-CustomHeader: value1,value2"
 ```
 
+## Env variables substitution
+
 migrator supports env variables substitution in config file. All patterns matching `${NAME}` will look for env variable `NAME`. Below are some common use cases:
 
 ```yaml
@@ -514,6 +525,51 @@ webHookHeaders:
   - "X-Security-Token: ${SECURITY_TOKEN}"
 ```
 
+## Source migrations
+
+Migrations can be read either from local disk or from S3 (I'm open to contributions to add more cloud storage options).
+
+### Local storage
+
+If `baseDir` property is a path (either relative or absolute) local storage implementation is used:
+
+```
+# relative path
+baseDir: test/migrations
+# absolute path
+baseDir: /project/migrations
+```
+
+### AWS S3
+
+If `baseDir` starts with `s3://` prefix, AWS S3 implementation is used. In such case the `baseDir` property is treated as a bucket name:
+
+```
+# S3 bucket
+baseDir: s3://lukasz-budnik-migrator-us-east-1
+```
+
+migrator uses official AWS SDK for Go and uses a well known [default credential provider chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). Please setup your env variables accordingly.
+
+## Supported databases
+
+Currently migrator supports the following databases and their flavours. Please review the Go driver implementation for information about supported features and how `dataSource` configuration property should look like:
+
+* PostgreSQL 9.3+ - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq
+  * PostgreSQL
+  * Amazon RDS PostgreSQL - PostgreSQL-compatible relational database built for the cloud
+  * Amazon Aurora PostgreSQL - PostgreSQL-compatible relational database built for the cloud
+  * Google CloudSQL PostgreSQL - PostgreSQL-compatible relational database built for the cloud
+* MySQL 5.6+ - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql
+  * MySQL
+  * MariaDB - enhanced near linearly scalable multi-master MySQL
+  * Percona - an enhanced drop-in replacement for MySQL
+  * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud
+  * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud
+  * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud
+* Microsoft SQL Server 2017 - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb
+  * Microsoft SQL Server
+
 # Customisation and legacy frameworks support
 
 migrator can be used with an already existing legacy DB migration framework.
@@ -575,25 +631,6 @@ When using migrator please remember that:
 * when adding a new tenant migrator creates a new DB schema and applies all tenant migrations and scripts - no need to apply them manually
 * single schemas are not created automatically, you must add initial migration with `create schema {schema}` SQL statement (see examples above)
 
-# Supported databases
-
-Currently migrator supports the following databases and their flavours:
-
-* PostgreSQL 9.3+ - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq
-  * PostgreSQL - original PostgreSQL server
-  * Amazon RDS PostgreSQL - PostgreSQL-compatible relational database built for the cloud
-  * Amazon Aurora PostgreSQL - PostgreSQL-compatible relational database built for the cloud
-  * Google CloudSQL PostgreSQL - PostgreSQL-compatible relational database built for the cloud
-* MySQL 5.6+ - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql
-  * MySQL - original MySQL server
-  * MariaDB - enhanced near linearly scalable multi-master MySQL
-  * Percona - an enhanced drop-in replacement for MySQL
-  * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud
-  * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud
-  * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud
-* Microsoft SQL Server 2017 - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb
-  * Microsoft SQL Server - original Microsoft SQL Server
-
 # Performance
 
 As a benchmarks I used 2 migrations frameworks:

loader/disk_loader.go

@@ -1,24 +1,20 @@
 package loader
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"io/ioutil"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	"fmt"
 
-	"github.com/lukaszbudnik/migrator/config"
 	"github.com/lukaszbudnik/migrator/types"
 )
 
 // diskLoader is struct used for implementing Loader interface for loading migrations from disk
 type diskLoader struct {
-	ctx    context.Context
-	config *config.Config
+	baseLoader
 }
 
 // GetSourceMigrations returns all migrations from disk
@@ -36,24 +32,17 @@ func (dl *diskLoader) GetSourceMigrations() []types.Migration {
 	tenantScriptsDirs := dl.getDirs(absBaseDir, dl.config.TenantScripts)
 
 	migrationsMap := make(map[string][]types.Migration)
-
 	dl.readFromDirs(migrationsMap, singleMigrationsDirs, types.MigrationTypeSingleMigration)
 	dl.readFromDirs(migrationsMap, tenantMigrationsDirs, types.MigrationTypeTenantMigration)
-	dl.readFromDirs(migrationsMap, singleScriptsDirs, types.MigrationTypeSingleScript)
-	dl.readFromDirs(migrationsMap, tenantScriptsDirs, types.MigrationTypeTenantScript)
+	dl.sortMigrations(migrationsMap, &migrations)
 
-	keys := make([]string, 0, len(migrationsMap))
-	for key := range migrationsMap {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
+	migrationsMap = make(map[string][]types.Migration)
+	dl.readFromDirs(migrationsMap, singleScriptsDirs, types.MigrationTypeSingleScript)
+	dl.sortMigrations(migrationsMap, &migrations)
 
-	for _, key := range keys {
-		ms := migrationsMap[key]
-		for _, m := range ms {
-			migrations = append(migrations, m)
-		}
-	}
+	migrationsMap = make(map[string][]types.Migration)
+	dl.readFromDirs(migrationsMap, tenantScriptsDirs, types.MigrationTypeTenantScript)
+	dl.sortMigrations(migrationsMap, &migrations)
 
 	return migrations
 }

loader/disk_loader_test.go

@@ -67,7 +67,7 @@ func TestDiskGetDiskMigrations(t *testing.T) {
 	loader := New(context.TODO(), &config)
 	migrations := loader.GetSourceMigrations()
 
-	assert.Len(t, migrations, 10)
+	assert.Len(t, migrations, 12)
 
 	assert.Contains(t, migrations[0].File, "test/migrations/config/201602160001.sql")
 	assert.Contains(t, migrations[1].File, "test/migrations/config/201602160002.sql")
@@ -77,6 +77,10 @@ func TestDiskGetDiskMigrations(t *testing.T) {
 	assert.Contains(t, migrations[5].File, "test/migrations/ref/201602160004.sql")
 	assert.Contains(t, migrations[6].File, "test/migrations/tenants/201602160004.sql")
 	assert.Contains(t, migrations[7].File, "test/migrations/tenants/201602160005.sql")
-	assert.Contains(t, migrations[8].File, "test/migrations/config-scripts/201912181227.sql")
-	assert.Contains(t, migrations[9].File, "test/migrations/tenants-scripts/201912181228.sql")
+	// SingleScripts are second to last
+	assert.Contains(t, migrations[8].File, "test/migrations/config-scripts/200012181227.sql")
+	// TenantScripts are last
+	assert.Contains(t, migrations[9].File, "test/migrations/tenants-scripts/200001181228.sql")
+	assert.Contains(t, migrations[10].File, "test/migrations/tenants-scripts/a.sql")
+	assert.Contains(t, migrations[11].File, "test/migrations/tenants-scripts/b.sql")
 }

loader/loader.go

@@ -2,6 +2,8 @@ package loader
 
 import (
 	"context"
+	"sort"
+	"strings"
 
 	"github.com/lukaszbudnik/migrator/config"
 	"github.com/lukaszbudnik/migrator/types"
@@ -17,5 +19,29 @@ type Factory func(context.Context, *config.Config) Loader
 
 // New returns new instance of Loader, currently DiskLoader is available
 func New(ctx context.Context, config *config.Config) Loader {
-	return &diskLoader{ctx, config}
+	if strings.HasPrefix(config.BaseDir, "s3://") {
+		return &s3Loader{baseLoader{ctx, config}}
+	}
+	return &diskLoader{baseLoader{ctx, config}}
+}
+
+// baseLoader is the base struct for implementing Loader interface
+type baseLoader struct {
+	ctx    context.Context
+	config *config.Config
+}
+
+func (bl *baseLoader) sortMigrations(migrationsMap map[string][]types.Migration, migrations *[]types.Migration) {
+	keys := make([]string, 0, len(migrationsMap))
+	for key := range migrationsMap {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		ms := migrationsMap[key]
+		for _, m := range ms {
+			*migrations = append(*migrations, m)
+		}
+	}
 }

loader/s3_loader.go

@@ -0,0 +1,119 @@
+package loader
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/lukaszbudnik/migrator/types"
+)
+
+// s3Loader is struct used for implementing Loader interface for loading migrations from AWS S3
+type s3Loader struct {
+	baseLoader
+}
+
+// GetSourceMigrations returns all migrations from AWS S3 location
+func (s3l *s3Loader) GetSourceMigrations() []types.Migration {
+	sess, err := session.NewSession()
+	if err != nil {
+		panic(err.Error())
+	}
+	client := s3.New(sess)
+	return s3l.doGetSourceMigrations(client)
+}
+
+func (s3l *s3Loader) doGetSourceMigrations(client s3iface.S3API) []types.Migration {
+	migrations := []types.Migration{}
+
+	singleMigrationsObjects := s3l.getObjectList(client, s3l.config.SingleMigrations)
+	tenantMigrationsObjects := s3l.getObjectList(client, s3l.config.TenantMigrations)
+	singleScriptsObjects := s3l.getObjectList(client, s3l.config.SingleScripts)
+	tenantScriptsObjects := s3l.getObjectList(client, s3l.config.TenantScripts)
+
+	migrationsMap := make(map[string][]types.Migration)
+	s3l.getObjects(client, migrationsMap, singleMigrationsObjects, types.MigrationTypeSingleMigration)
+	s3l.getObjects(client, migrationsMap, tenantMigrationsObjects, types.MigrationTypeTenantMigration)
+	s3l.sortMigrations(migrationsMap, &migrations)
+
+	migrationsMap = make(map[string][]types.Migration)
+	s3l.getObjects(client, migrationsMap, singleScriptsObjects, types.MigrationTypeSingleScript)
+	s3l.sortMigrations(migrationsMap, &migrations)
+
+	migrationsMap = make(map[string][]types.Migration)
+	s3l.getObjects(client, migrationsMap, tenantScriptsObjects, types.MigrationTypeTenantScript)
+	s3l.sortMigrations(migrationsMap, &migrations)
+
+	return migrations
+}
+
+func (s3l *s3Loader) getObjectList(client s3iface.S3API, prefixes []string) []*string {
+	objects := []*string{}
+
+	bucket := strings.Replace(s3l.config.BaseDir, "s3://", "", 1)
+
+	for _, prefix := range prefixes {
+
+		input := &s3.ListObjectsV2Input{
+			Bucket:  aws.String(bucket),
+			Prefix:  aws.String(prefix),
+			MaxKeys: aws.Int64(1000),
+		}
+
+		pageNum := 0
+		err := client.ListObjectsV2Pages(input,
+			func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+				pageNum++
+				for _, o := range page.Contents {
+					objects = append(objects, o.Key)
+				}
+
+				return pageNum <= 10
+			})
+
+		if err != nil {
+			panic(err.Error())
+		}
+	}
+
+	return objects
+}
+
+func (s3l *s3Loader) getObjects(client s3iface.S3API, migrationsMap map[string][]types.Migration, objects []*string, migrationType types.MigrationType) {
+	bucket := strings.Replace(s3l.config.BaseDir, "s3://", "", 1)
+
+	objectInput := &s3.GetObjectInput{Bucket: aws.String(bucket)}
+	for _, o := range objects {
+		objectInput.Key = o
+		objectOutput, err := client.GetObject(objectInput)
+		if err != nil {
+			panic(err.Error())
+		}
+		buf := new(bytes.Buffer)
+		buf.ReadFrom(objectOutput.Body)
+		contents := buf.String()
+
+		hasher := sha256.New()
+		hasher.Write([]byte(contents))
+		file := fmt.Sprintf("%s/%s", s3l.config.BaseDir, *o)
+		from := strings.LastIndex(file, "/")
+		sourceDir := file[0:from]
+		name := file[from+1:]
+		m := types.Migration{Name: name, SourceDir: sourceDir, File: file, MigrationType: migrationType, Contents: string(contents), CheckSum: hex.EncodeToString(hasher.Sum(nil))}
+
+		e, ok := migrationsMap[m.Name]
+		if ok {
+			e = append(e, m)
+		} else {
+			e = []types.Migration{m}
+		}
+		migrationsMap[m.Name] = e
+
+	}
+}