WIP: Store migrates in db

.github/workflows/ci.yaml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   lint:
+    if: false # disable linting for infobloxopen/migrate
     name: lint
     runs-on: ubuntu-latest
     steps:
@@ -31,7 +32,7 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: Run test
-        run: make test COVERAGE_DIR=/tmp/coverage
+        run: make test COVERAGE_DIR=/tmp/coverage DATABASE="postgres mysql clickhouse mongodb pgx pgx5 rqlite sqlite sqlite3"
 
       - name: Send goveralls coverage
         uses: shogo82148/actions-goveralls@v1
@@ -51,11 +52,11 @@ jobs:
 
   goreleaser:
     name: Release a new version
-    needs: [lint, test]
+    needs: [test]
     runs-on: ubuntu-latest
     environment: GoReleaser
     # This job only runs when
-    # 1. When the previous `lint` and `test` jobs has completed successfully
+    # 1. When the previous `test` job has completed successfully
     # 2. When the repository is not a fork, i.e. it will only run on the official golang-migrate/migrate
     # 3. When the workflow is triggered by a tag with `v` prefix
     if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }}

Dockerfile

@@ -3,7 +3,7 @@ ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
 
-WORKDIR /go/src/github.com/golang-migrate/migrate
+WORKDIR /go/src/github.com/infobloxopen/migrate
 
 ENV GO111MODULE=on
 
@@ -15,12 +15,11 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.21
+FROM gcr.io/distroless/static:nonroot
 
-RUN apk add --no-cache ca-certificates
-
-COPY --from=builder /go/src/github.com/golang-migrate/migrate/build/migrate.linux-386 /usr/local/bin/migrate
-RUN ln -s /usr/local/bin/migrate /migrate
+COPY --from=builder /go/src/github.com/infobloxopen/migrate/cmd/migrate/config /cli/config/
+COPY --from=builder /go/src/github.com/infobloxopen/migrate/build/migrate.linux-386 /migrate
+COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs/
 
 ENTRYPOINT ["migrate"]
 CMD ["--help"]

Dockerfile.github-actions

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:latest
 
 RUN apk add --no-cache ca-certificates
 

Jenkinsfile

@@ -0,0 +1,63 @@
+
+// This library defines the isPrBuild, prepareBuild and finalizeBuild methods
+@Library('jenkins.shared.library') _
+
+pipeline {
+  agent {
+    label 'ubuntu_docker_label'
+  }
+  tools {
+    go "Go 1.24.2"
+  }
+  options {
+    checkoutToSubdirectory('src/github.com/infobloxopen/migrate')
+  }
+  environment {
+    GOPATH = "$WORKSPACE"
+    DIRECTORY = "src/github.com/infobloxopen/migrate"
+  }
+
+  stages {
+    stage("Setup") {
+      steps {
+        // prepareBuild is one of the Secure CICD helper methods
+        prepareBuild()
+      }
+    }
+    stage("Unit Tests") {
+      steps {
+        dir("$DIRECTORY") {
+          // sh "make test"
+        }
+      }
+    }
+    stage("Build Image") {
+      steps {
+        withDockerRegistry([credentialsId: "${env.JENKINS_DOCKER_CRED_ID}", url: ""]) {
+          dir("$DIRECTORY") {
+            sh "make build"
+          }
+        }
+      }
+    }
+  }
+  post {
+    success {
+      // finalizeBuild is one of the Secure CICD helper methods
+      dir("$DIRECTORY") {
+        finalizeBuild(
+          sh(
+            script: 'make list-of-images',
+            returnStdout: true
+          )
+        )
+      }
+    }
+    cleanup {
+      dir("$DIRECTORY") {
+        sh "make clean || true"
+      }
+      cleanWs()
+    }
+  }
+}

Makefile

@@ -1,14 +1,12 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
 DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
-VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
+BUILD_NUMBER ?= 0
+VERSION ?= $(shell git describe --tags --long --dirty=-unsupported 2>/dev/null | cut -c 2-)-j$(BUILD_NUMBER)
 TEST_FLAGS ?=
 REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)")
 COVERAGE_DIR ?= .coverage
 
-build:
-	CGO_ENABLED=0 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate
-
 build-docker:
 	CGO_ENABLED=0 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate
 
@@ -24,6 +22,17 @@ build-cli: clean
 	cd ./cli/build && shasum -a 256 * > sha256sum.txt
 	cat ./cli/build/sha256sum.txt
 
+build:
+	docker build --pull --build-arg VERSION=$(VERSION) . -t infoblox/migrate -t infoblox/migrate:$(VERSION)
+
+docker-push:
+	docker push infoblox/migrate:$(VERSION)
+
+show-image-version:
+	echo $(VERSION)
+
+list-of-images:
+	@echo "infoblox/migrate:$(VERSION)"
 
 clean:
 	-rm -r ./cli/build
@@ -117,4 +126,3 @@ endef
 
 SHELL = /bin/sh
 RAND = $(shell echo $$RANDOM)
-

cli/main.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/golang-migrate/migrate/v4/internal/cli"
+import (
+	"github.com/golang-migrate/migrate/v4/internal/cli"
+)
 
 // Deprecated, please use cmd/migrate
 func main() {

cmd/migrate/.gitignore

@@ -0,0 +1 @@
+migrate

cmd/migrate/config.go

@@ -0,0 +1,42 @@
+package main
+
+import "github.com/spf13/pflag"
+
+const (
+	// configuration defaults support local development (i.e. "go run ...")
+	defaultDatabaseDSN      = ""
+	defaultDatabaseDriver   = "postgres"
+	defaultDatabaseAddress  = "0.0.0.0:5432"
+	defaultDatabaseName     = ""
+	defaultDatabaseUser     = "postgres"
+	defaultDatabasePassword = "postgres"
+	defaultDatabaseSSL      = "disable"
+	defaultConfigDirectory  = "/cli/config"
+)
+
+var (
+	// define flag overrides
+	flagHelp           = pflag.Bool("help", false, "Print usage")
+	flagVersion        = pflag.String("version", Version, "Print version")
+	flagLoggingVerbose = pflag.Bool("verbose", true, "Print verbose logging")
+	flagPrefetch       = pflag.Uint("prefetch", 10, "Number of migrations to load in advance before executing")
+	flaglockTimeout    = pflag.Uint("lock-timeout", 15, "Allow N seconds to acquire database lock")
+
+	flagDatabaseDSN      = pflag.String("database.dsn", defaultDatabaseDSN, "database connection string")
+	flagDatabaseDriver   = pflag.String("database.driver", defaultDatabaseDriver, "database driver")
+	flagDatabaseAddress  = pflag.String("database.address", defaultDatabaseAddress, "address of the database")
+	flagDatabaseName     = pflag.String("database.name", defaultDatabaseName, "name of the database")
+	flagDatabaseUser     = pflag.String("database.user", defaultDatabaseUser, "database username")
+	flagDatabasePassword = pflag.String("database.password", defaultDatabasePassword, "database password")
+	flagDatabaseSSL      = pflag.String("database.ssl", defaultDatabaseSSL, "database ssl mode")
+
+	flagSource = pflag.String("source", "", "Location of the migrations (driver://url)")
+	flagPath   = pflag.String("path", "", "Shorthand for -source=file://path")
+
+	flagConfigDirectory = pflag.String("config.source", defaultConfigDirectory, "directory of the configuration file")
+	flagConfigFile      = pflag.String("config.file", "", "configuration file name without extension")
+
+	// goto command flags
+	flagDirty     = pflag.Bool("force-dirty-handling", false, "force the handling of dirty database state")
+	flagMountPath = pflag.String("cache-dir", "", "path to the cache-dir which is used to copy the migration files")
+)

cmd/migrate/config/defaults.yaml

@@ -0,0 +1,14 @@
+help: false
+version: false
+verbose: true
+prefetch: 10
+lockTimeout: 15
+path: "/atlas-migrations/migrations"
+#source: "file:///atlas-migrations/migrations"
+database:
+  driver: postgres
+  address: postgres:5432
+  name: app_db
+  user: postgres
+  password: postgres
+  ssl: disable

cmd/migrate/main.go

@@ -1,6 +1,39 @@
 package main
 
-import "github.com/golang-migrate/migrate/v4/internal/cli"
+import (
+	"log"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v4/internal/cli"
+	"github.com/infobloxopen/hotload"
+	_ "github.com/infobloxopen/hotload/fsnotify"
+	"github.com/jackc/pgx/v4/stdlib"
+	"github.com/lib/pq"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+)
+
+func init() {
+	pflag.Parse()
+	viper.BindPFlags(pflag.CommandLine)
+	viper.AutomaticEnv()
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AddConfigPath(viper.GetString("config.source"))
+	if viper.GetString("config.file") != "" {
+		viper.SetConfigName(viper.GetString("config.file"))
+		if err := viper.ReadInConfig(); err != nil {
+			log.Fatalf("cannot load configuration: %v", err)
+		}
+	}
+	// logrus formatter
+	customFormatter := new(logrus.JSONFormatter)
+	logrus.SetFormatter(customFormatter)
+
+	hotload.RegisterSQLDriver("pgx", stdlib.GetDefaultDriver())
+	hotload.RegisterSQLDriver("postgres", pq.Driver{})
+	hotload.RegisterSQLDriver("postgresql", pq.Driver{})
+}
 
 func main() {
 	cli.Main(Version)