Implemets Issue #33: Add support for MySQL upgrade

Version 1.10.0 adds command 'admin upgrade from_sandbox to_sandbox'
Also added upgrade test.

Author: dataclouder

Diff for: .build/COMPATIBLE_VERSION

@@ -1 +1 @@
-1.9.0
+1.10.0

Diff for: .build/VERSION

@@ -1 +1 @@
-1.9.0
+1.10.0

Diff for: .build/create-version-source-file.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"bufio"
+	//"bufio"
 	"fmt"
 	"github.com/datacharmer/dbdeployer/common"
 	"os"
@@ -25,7 +25,6 @@ func main() {
 		if !common.DirExists(templates_dir) {
 			common.Exit(1, "Directory .build/ not found")
 		}
-
 	}
 	version_dest_file := "common/version.go"
 
@@ -51,6 +50,7 @@ func main() {
 		"Timestamp":             time.Now().Format("2006-01-02 15:04"),
 	}
 	version_code := common.Tprintf(template, data)
+	/*
 	file, err := os.Open(version_dest_file)
 	if err != nil {
 		common.Exit(1, fmt.Sprintf("error opening file %s", version_dest_file))
@@ -61,5 +61,7 @@ func main() {
 	if err != nil {
 		common.Exit(1, fmt.Sprintf("error writing to file %s", version_dest_file))
 	}
-	fmt.Printf("Written %d bytes into %s\n", written, version_dest_file)
+	*/
+	common.WriteString(version_code, version_dest_file)
+	// fmt.Printf("Written %d bytes into %s\n", written, version_dest_file)
 }

Diff for: .build/set_version.sh

@@ -2,7 +2,6 @@ language: go
 
 go:
   - "1.10.x"
-  - master
 
 os: linux
 

Diff for: .travis.yml

@@ -1,3 +1,6 @@
+1.10.0	27-Aug-2018
+	- Fixed Issue#32 "Error log was not created in data directory"
+	- Added command 'dbdeployer admin upgrade sandbox1 sandbox2' (Issue#33)
 1.9.0	12-Aug-2018
 	- Implemented Issue#31 "Add feature to extract and merge a mysql-shell
 	  tarball into a regular one"

Diff for: Changelog

@@ -3,7 +3,7 @@
 [DBdeployer](https://github.com/datacharmer/dbdeployer) is a tool that deploys MySQL database servers easily.
 This is a port of [MySQL-Sandbox](https://github.com/datacharmer/mysql-sandbox), originally written in Perl, and re-designed from the ground up in [Go](https://golang.org). See the [features comparison](https://github.com/datacharmer/dbdeployer/blob/master/docs/features.md) for more detail.
 
-Documentation updated for version 1.9.0 (14-Aug-2018 08:30 UTC)
+Documentation updated for version 1.10.0 (26-Aug-2018 21:57 UTC)
 
 [![Build Status](https://travis-ci.org/datacharmer/dbdeployer.svg "Travis CI status")](https://travis-ci.org/datacharmer/dbdeployer)
 
@@ -15,7 +15,7 @@ Get the one for your O.S. from [dbdeployer releases](https://github.com/datachar
 
 For example:
 
-    $ VERSION=1.9.0
+    $ VERSION=1.10.0
     $ OS=linux
     $ origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION
     $ wget $origin/dbdeployer-$VERSION.$OS.tar.gz
@@ -50,7 +50,7 @@ For example:
 The program doesn't have any dependencies. Everything is included in the binary. Calling *dbdeployer* without arguments or with ``--help`` will show the main help screen.
 
     $ dbdeployer --version
-    dbdeployer version 1.9.0
+    dbdeployer version 1.10.0
 
 
     $ dbdeployer -h
@@ -819,25 +819,62 @@ The lock can also be reverted using
 
     $ dbdeployer admin unlock sandbox_name
 
+## Sandbox upgrade
+
+dbdeployer 1.10.0 introduces upgrades:
+
+    $ dbdeployer admin upgrade -h
+    Upgrades a sandbox to a newer version.
+    The sandbox with the new version must exist already.
+    The data directory of the old sandbox will be moved to the new one.
+    
+    Usage:
+      dbdeployer admin upgrade sandbox_name newer_sandbox [flags]
+    
+    Examples:
+    dbdeployer admin upgrade msb_8_0_11 msb_8_0_12
+    
+    Flags:
+      -h, --help   help for upgrade
+    
+    
+
+To perform an upgrade, the following conditions myst be met:
+
+* Both sandboxes must be **single** deployments.
+* The older version must be one major version behind (5.6.x to 5.7.x, or 5.7.x to 8.0.x, but not 5.6.x to 8.0.x) or same major version but different revision (e.g. 5.7.22 to 5.7.23)
+* The newer version must have been already deployed.
+* The newer version must have mysql_upgrade in its base directory (e.g $SANDBOX_BINARY/5.7.23/bin)
+
+dbdeployer checks all the conditions, then
+
+1. stops both databases;
+2. renames the data directory of the newer version;
+3. moves the data directory of the older version under the newer sandbox;
+4. restarts the newer version;
+5. runs mysql_upgrade.
+
+The older version is, at this point, not operational anymore, and can be deleted.
+
 ## Compiling dbdeployer
 
 Should you need to compile your own binaries for dbdeployer, follow these steps:
 
 1. Make sure you have go installed in your system, and that the ``$GOPATH`` variable is set.
 2. Run ``go get -u github.com/datacharmer/dbdeployer``.  This will import all the code that is needed to build dbdeployer.
 3. Change directory to ``$GOPATH/src/github.com/datacharmer/dbdeployer``.
-4. Run ``./build.sh {linux|OSX} 1.9.0``
-5. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./build.sh {linux|OSX} 1.9.0``
+4. Run ``./build.sh {linux|OSX} 1.10.0``
+5. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./build.sh {linux|OSX} 1.10.0``
 
 ## Generating additional documentation
 
 Between this file and [the API API list](https://github.com/datacharmer/dbdeployer/blob/master/docs/API/API-1.1.md), you have all the existing documentation for dbdeployer.
 Should you need additional formats, though, dbdeployer is able to generate them on-the-fly. Tou will need the docs-enabled binaries: in the distribution list, you will find:
 
-* dbdeployer-1.9.0-docs.linux.tar.gz
-* dbdeployer-1.9.0-docs.osx.tar.gz
-* dbdeployer-1.9.0.linux.tar.gz
-* dbdeployer-1.9.0.osx.tar.gz
+* dbdeployer-1.10.0-docs.linux.tar.gz
+* dbdeployer-1.10.0-docs.osx.tar.gz
+* dbdeployer-1.10.0.linux.tar.gz
+* dbdeployer-1.10.0.osx.tar.gz
 
 The executables containing ``-docs`` in their name have the same capabilities of the regular ones, but in addition they can run the *hidden* command ``tree``, with alias ``docs``.
 

Diff for: README.md

@@ -109,6 +109,11 @@ case $target in
 	    (set -x
         env GOOS=darwin GOARCH=386 go build $docs_flags -o $executable .
         )
+        if [ "$?" != "0" ]
+        then
+            echo "ERROR during build!"
+            exit 1
+        fi
         tar -c $executable | gzip -c > ${executable}.tar.gz
         build_sort Darwin
     ;;

Diff for: build.sh

@@ -152,6 +152,121 @@ func UnlockSandbox(cmd *cobra.Command, args []string) {
 	}
 }
 
+func UpgradeSandbox(sandbox_dir, old_sandbox, new_sandbox string) {
+	var possible_upgrades = map[string]string{
+		"5.0" : "5.1",
+		"5.1" : "5.5",
+		"5.5" : "5.6",
+		"5.6" : "5.7",
+		"5.7" : "8.0",
+		"8.0" : "8.0",
+	}
+	err := os.Chdir(sandbox_dir)
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error: can't change directory to %s", sandbox_dir))
+	}
+	scripts := []string{"start", "stop", "my"}
+	for _, dir := range []string{old_sandbox, new_sandbox} {
+		if !common.DirExists(dir) {
+			common.Exit(1, fmt.Sprintf("Error: Directory %s not found in %s", dir, sandbox_dir))
+		}
+		for _, script := range scripts {
+			if !common.ExecExists( dir + "/" + script) {
+				common.Exit(1, fmt.Sprintf("Error: script %s not found in %s", script, dir),
+				"The upgrade only works between SINGLE deployments")
+			}
+		}
+	}
+	new_sbdesc := common.ReadSandboxDescription(new_sandbox)
+	old_sbdesc := common.ReadSandboxDescription(old_sandbox)
+	mysql_upgrade := new_sbdesc.Basedir + "/bin/mysql_upgrade"
+	if !common.ExecExists(mysql_upgrade) {
+		common.WriteString("", new_sandbox + "/no_upgrade")
+		common.Exit(0, fmt.Sprintf("mysql_upgrade not found in %s. Upgrade is not possible", new_sbdesc.Basedir))
+	}
+	new_version_list := common.VersionToList(new_sbdesc.Version)
+	new_major := new_version_list[0]
+	new_minor := new_version_list[1]
+	new_rev := new_version_list[2]
+	old_version_list := common.VersionToList(old_sbdesc.Version)
+	old_major := old_version_list[0]
+	old_minor := old_version_list[1]
+	old_rev := old_version_list[2]
+	new_upgrade_version := fmt.Sprintf("%d.%d", new_version_list[0], new_version_list[1])
+	old_upgrade_version := fmt.Sprintf("%d.%d", old_version_list[0], old_version_list[1])
+	if old_major == 10 || new_major == 10 {
+		common.Exit(1, "Upgrade from and to MariaDB is not supported")
+	}
+	if common.GreaterOrEqualVersion(old_sbdesc.Version, new_version_list) {
+		common.Exit(1, fmt.Sprintf("Version %s must be greater than %s", new_upgrade_version, old_upgrade_version))
+	}
+	can_be_upgraded := false
+	if old_major < new_major {
+		can_be_upgraded = true
+	} else {
+		if old_major == new_major && old_minor < new_minor {
+			can_be_upgraded = true
+		} else {
+			if old_major == new_major && old_minor == new_minor && old_rev < new_rev {
+				can_be_upgraded = true
+			}
+		}
+	}
+	if !can_be_upgraded {
+		common.Exit(1, fmt.Sprintf("Version %s can only be upgraded to %s or to the same version with a higher revision", old_upgrade_version, possible_upgrades[old_upgrade_version]))
+	}
+	new_sandbox_old_data :=  new_sandbox + "/data-" + new_sandbox
+	if common.DirExists(new_sandbox_old_data) {
+		common.Exit(1, fmt.Sprintf("Sandbox %s is already the upgrade from an older version", new_sandbox))
+	}
+	err, _ = common.Run_cmd(old_sandbox + "/stop")
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while stopping sandbox %s", old_sandbox))
+	}
+	err, _ = common.Run_cmd(new_sandbox + "/stop")
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while stopping sandbox %s", new_sandbox))
+	}
+	mv_args := []string{new_sandbox + "/data", new_sandbox_old_data}
+	err, _ = common.Run_cmd_with_args("mv", mv_args)
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while moving data directory in sandbox %s", new_sandbox))
+	}
+
+	mv_args = []string{old_sandbox + "/data", new_sandbox + "/data" }
+	err, _ = common.Run_cmd_with_args("mv", mv_args)
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while moving data directory from sandbox %s to %s", old_sandbox, new_sandbox))
+	}
+	fmt.Printf("Data directory %s/data moved to %s/data \n", old_sandbox, new_sandbox)
+
+	err, _ = common.Run_cmd(new_sandbox + "/start")
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while starting sandbox %s", new_sandbox))
+	}
+	upgrade_args := []string{"sql_upgrade"}
+	err, _ = common.Run_cmd_with_args(new_sandbox + "/my", upgrade_args)
+	if err != nil {
+		common.Exit(1, fmt.Sprintf("Error while running mysql_upgrade in %s", new_sandbox))
+	}
+	fmt.Println("")
+	fmt.Printf("The data directory from %s/data is preserved in %s\n", new_sandbox, new_sandbox_old_data)
+	fmt.Printf("The data directory from %s/data is now used in %s/data\n", old_sandbox, new_sandbox)
+	fmt.Printf("%s is not operational and can be deleted\n", old_sandbox)
+}
+
+func RunUpgradeSandbox(cmd *cobra .Command, args []string) {
+	if len(args) < 2 {
+		common.Exit(1,
+			"'upgrade' requires the name of two sandboxes ",
+			"Example: dbdeployer admin upgrade msb_5_7_23 msb_8_0_12")
+	}
+	old_sandbox := args[0]
+	new_sandbox := args[1]
+	sandbox_dir := GetAbsolutePathFromFlag(cmd, "sandbox-home")
+	UpgradeSandbox(sandbox_dir, old_sandbox, new_sandbox)
+}
+
 var (
 	adminCmd = &cobra.Command{
 		Use:     "admin",
@@ -177,10 +292,21 @@ Users can still delete locked sandboxes manually.`,
 		Long:    `Removes lock, allowing deletion of a given sandbox`,
 		Run:     UnlockSandbox,
 	}
+	adminUpgradeCmd = &cobra.Command{
+		Use:     "upgrade sandbox_name newer_sandbox",
+		Short:   "Upgrades a sandbox to a newer version",
+		Long:    `Upgrades a sandbox to a newer version.
+The sandbox with the new version must exist already.
+The data directory of the old sandbox will be moved to the new one.`,
+		Example: "dbdeployer admin upgrade msb_8_0_11 msb_8_0_12",
+		Run:     RunUpgradeSandbox,
+	}
+
 )
 
 func init() {
 	rootCmd.AddCommand(adminCmd)
 	adminCmd.AddCommand(adminLockCmd)
 	adminCmd.AddCommand(adminUnlockCmd)
+	adminCmd.AddCommand(adminUpgradeCmd)
 }

Diff for: cmd/admin.go

@@ -159,7 +159,8 @@ func WriteStrings(lines []string, filename string, termination string) error {
 
 	w := bufio.NewWriter(file)
 	for _, line := range lines {
-		fmt.Fprintln(w, line+termination)
+		//fmt.Fprintln(w, line+termination)
+		fmt.Fprintf(w, "%s", line+termination)
 	}
 	return w.Flush()
 }

Diff for: common/fileutil.go

@@ -16,11 +16,11 @@
 package common
 
 // This file was generated during build. Do not edit.
-// Build time: 2018-08-15 05:55
+// Build time: 2018-08-26 23:55
 
-var VersionDef string = "1.9.0" // 2018-08-12
+var VersionDef string = "1.10.0" // 2018-08-26
 
 // Compatible version is the version used to mark compatible archives (templates, configuration).
 // It is usually major.minor.0, except when we are at version 0.x, when
 // every revision may bring incompatibility
-var CompatibleVersion string = "1.9.0" // 2018-08-12
+var CompatibleVersion string = "1.10.0" // 2018-08-26