diff --git a/commands/project.go b/commands/project.go index 692ca30..fc4dcde 100644 --- a/commands/project.go +++ b/commands/project.go @@ -76,7 +76,7 @@ func (cmd *Project) GetScriptsAsSubcommands(otherSubcommands []cli.Command) []cl func (cmd *Project) Run(c *cli.Context) error { cmd.out.Verbose("Loaded project configuration from %s", cmd.Config.Path) if cmd.Config.Scripts == nil { - cmd.out.Channel.Error.Fatal("There are no scripts discovered in: %s", cmd.Config.File) + cmd.out.Channel.Error.Fatal(fmt.Sprintf("There are no scripts discovered in: %s", cmd.Config.File)) } key := strings.TrimPrefix(c.Command.Name, "run:") diff --git a/commands/project_script.go b/commands/project_script.go index 85a9a8c..d33053a 100644 --- a/commands/project_script.go +++ b/commands/project_script.go @@ -10,7 +10,7 @@ import ( "github.com/phase2/rig/util" ) -// ProjectScript wrapps the evaluation of project scripts. +// ProjectScript wraps the evaluation of project scripts. // It mimics command struct except with unexported values. type ProjectScript struct { out *util.RigLogger diff --git a/commands/project_sync.go b/commands/project_sync.go index b603255..389fbcd 100644 --- a/commands/project_sync.go +++ b/commands/project_sync.go @@ -8,12 +8,10 @@ import ( "os/exec" "path" "path/filepath" - "regexp" "runtime" "strings" "time" - "github.com/hashicorp/go-version" "github.com/urfave/cli" "gopkg.in/yaml.v2" @@ -44,6 +42,7 @@ func (cmd *ProjectSync) Commands() []cli.Command { start := cli.Command{ Name: "sync:start", Aliases: []string{"sync"}, + Category: "File Sync", Usage: "Start a Unison sync on local project directory.", Description: "Volume name will be discovered in the following order: outrigger project config > docker-compose file > current directory name", Flags: []cli.Flag{ @@ -73,6 +72,7 @@ func (cmd *ProjectSync) Commands() []cli.Command { } stop := cli.Command{ Name: "sync:stop", + Category: "File Sync", Usage: "Stops a Unison sync on local project directory.", Description: "Volume name will be discovered in the following order: outrigger project config > docker-compose file > current directory name", Flags: []cli.Flag{ @@ -86,35 +86,226 @@ func (cmd *ProjectSync) Commands() []cli.Command { Before: cmd.Before, Action: cmd.RunStop, } - - return []cli.Command{start, stop} + name := cli.Command{ + Name: "sync:name", + Category: "File Sync", + Usage: "Retrieves the name used for the Unison volume and container.", + Description: "This will perform the same name discovery used by sync:start and returns it to ease scripting.", + Flags: []cli.Flag{ + // Override the local sync path. + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "Specify the location in the local filesystem to be synced. If not used it will look for the directory of project configuration or fall back to current working directory. Use '--dir=.' to guarantee current working directory is used.", + }, + }, + Before: cmd.Before, + Action: cmd.RunName, + } + check := cli.Command{ + Name: "sync:check", + Category: "File Sync", + Usage: "Run doctor checks on the state of your unison file sync.", + Description: "This is intended to facilitate easy verification whether the filesync is down.", + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "initial-sync-timeout", + Value: 120, + Usage: "Maximum amount of time in seconds to allow for detecting each of start of the Unison container and start of initial sync. If you encounter failures detecting initial sync increasing this value may help. Search for sync on http://docs.outrigger.sh/faq/troubleshooting/ (not needed on linux)", + EnvVar: "RIG_PROJECT_SYNC_TIMEOUT", + }, + // Arbitrary sleep length but anything less than 3 wasn't catching + // ongoing very quick file updates during a test + cli.IntFlag{ + Name: "initial-sync-wait", + Value: 5, + Usage: "Time in seconds to wait between checks to see if initial sync has finished. (not needed on linux)", + EnvVar: "RIG_PROJECT_INITIAL_SYNC_WAIT", + }, + // Override the local sync path. + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "Specify the location in the local filesystem to be synced. If not used it will look for the directory of project configuration or fall back to current working directory. Use '--dir=.' to guarantee current working directory is used.", + }, + }, + Before: cmd.Before, + Action: cmd.RunCheck, + } + purge := cli.Command{ + Name: "sync:purge", + Category: "File Sync", + Usage: "Purges an existing sync volume for the current project/directory.", + Description: "This goes beyond sync:stop to remove the Docker plumbing of the file sync for a clean restart.", + Flags: []cli.Flag{ + // Override the local sync path. + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "Specify the location in the local filesystem to be synced. If not used it will look for the directory of project configuration or fall back to current working directory. Use '--dir=.' to guarantee current working directory is used.", + }, + }, + Before: cmd.Before, + Action: cmd.RunPurge, + } + return []cli.Command{start, stop, name, check, purge} } // RunStart executes the `rig project sync:start` command to start the Unison sync process. func (cmd *ProjectSync) RunStart(ctx *cli.Context) error { + volumeName, workingDir, err := cmd.initializeSettings(ctx.String("dir")) + if err != nil { + return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + } + + switch platform := runtime.GOOS; platform { + case "linux": + cmd.out.Verbose("Setting up local volume: %s", volumeName) + return cmd.SetupBindVolume(volumeName, workingDir) + default: + cmd.out.Verbose("Starting sync with volume: %s", volumeName) + return cmd.StartUnisonSync(ctx, volumeName, cmd.Config, workingDir) + } +} + +// RunStop executes the `rig project sync:stop` command to shut down and unison containers +func (cmd *ProjectSync) RunStop(ctx *cli.Context) error { + if util.IsLinux() { + return cmd.Success("No Unison container to stop, using local bind volume") + } + cmd.out.Spin(fmt.Sprintf("Stopping Unison container")) + + volumeName, _, err := cmd.initializeSettings(ctx.String("dir")) + if err != nil { + return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + } + + cmd.out.Spin(fmt.Sprintf("Stopping Unison container (%s)", volumeName)) + if err := util.Command("docker", "container", "stop", volumeName).Run(); err != nil { + return cmd.Failure(err.Error(), "SYNC-CONTAINER-FAILURE", 13) + } + + return cmd.Success(fmt.Sprintf("Unison container '%s' stopped", volumeName)) +} + +// RunName provides the name of the sync volume and container. This is made available to facilitate scripting. +func (cmd *ProjectSync) RunName(ctx *cli.Context) error { + name, _, err := cmd.initializeSettings(ctx.String("dir")) + if err != nil { + return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + } + + fmt.Println(name) + return nil +} + +// RunCheck performs a doctor-like examination of the file sync health. +func (cmd *ProjectSync) RunCheck(ctx *cli.Context) error { + cmd.out.Spin("Preparing test of unison filesync...") + volumeName, workingDir, err := cmd.initializeSettings(ctx.String("dir")) + if err != nil { + return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + } + cmd.out.Info("Ready to begin unison test") + cmd.out.Spin("Checking for unison container...") + if running := util.ContainerRunning(volumeName); !running { + return cmd.Failure(fmt.Sprintf("Unison container (%s) is not running", volumeName), "SYNC-CHECK-FAILED", 13) + } + cmd.out.Info("Unison container found: %s", volumeName) + cmd.out.Spin("Check unison container process is listening...") + if _, err := cmd.WaitForUnisonContainer(volumeName, ctx.Int("initial-sync-timeout")); err != nil { + cmd.out.Error("Unison process not listening") + return cmd.Failure(err.Error(), "SYNC-CHECK-FAILED", 13) + } + cmd.out.Info("Unison process is listening") + + // Determine if sync progress can be tracked. + //cmd.out.Spin("Syncing a test file...") + cmd.out.Info("Preparing live file sync test") + var logFile = cmd.LogFileName(volumeName) + if err := cmd.WaitForSyncInit(logFile, workingDir, ctx.Int("initial-sync-timeout"), ctx.Int("initial-sync-wait")); err != nil { + return cmd.Failure(err.Error(), "UNISON-SYNC-FAILED", 13) + } + + // Sidestepping the notification so rig sync:check can be run as a background process. + cmd.out.Info("Sync check completed successfully") + return nil +} + +// RunPurge cleans out the project sync state. +func (cmd *ProjectSync) RunPurge(ctx *cli.Context) error { + if util.IsLinux() { + return cmd.Success("No Unison process to clean up.") + } + + volumeName, workingDir, err := cmd.initializeSettings(ctx.String("dir")) + if err != nil { + return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + } + + cmd.out.Spin("Checking for unison container...") + if running := util.ContainerRunning(volumeName); running { + cmd.out.Spin(fmt.Sprintf("Stopping Unison container (%s)", volumeName)) + if stopErr := util.Command("docker", "container", "stop", volumeName).Run(); stopErr != nil { + cmd.out.Warn("Could not stop unison container (%s): Maybe it's already stopped?", volumeName) + } else { + cmd.out.Info("Stopped unison container (%s)", volumeName) + } + } else { + cmd.out.Info("No running unison container.") + } + + logFile := cmd.LogFileName(volumeName) + cmd.out.Spin(fmt.Sprintf("Removing unison log file: %s", logFile)) + if util.FileExists(logFile, workingDir) { + if removeErr := util.RemoveFile(logFile, workingDir); removeErr != nil { + cmd.out.Error("Could not remove unison log file: %s: %s", logFile, removeErr.Error()) + } else { + cmd.out.Info("Removed unison log file: %s", logFile) + } + } else { + cmd.out.Info("Log file does not exist") + } + + // Remove sync fragment files. + cmd.out.Spin("Removing .unison directories") + if removeGlobErr := util.RemoveFileGlob("*.unison*", workingDir, cmd.out); removeGlobErr != nil { + cmd.out.Warning("Could not remove .unison directories: %s", removeGlobErr) + } else { + cmd.out.Info("Removed all .unison directories") + } + + cmd.out.Spin(fmt.Sprintf("Removing sync volume: %s", volumeName)) + // @TODO capture the volume rm error text to display to user! + out, rmErr := util.Command("docker", "volume", "rm", "--force", volumeName).CombinedOutput() + if rmErr != nil { + fmt.Println(rmErr.Error()) + return cmd.Failure(string(out), "SYNC-VOLUME-REMOVE-FAILURE", 13) + } + + cmd.out.Info("Sync volume (%s) removed", volumeName) + return nil +} + +// initializeSettings pulls together the configuration and contextual settings +// used for all sync operations. +func (cmd *ProjectSync) initializeSettings(dir string) (string, string, error) { cmd.Config = NewProjectConfig() if cmd.Config.NotEmpty() { cmd.out.Verbose("Loaded project configuration from %s", cmd.Config.Path) } // Determine the working directory for CWD-sensitive operations. - var workingDir, err = cmd.DeriveLocalSyncPath(cmd.Config, ctx.String("dir")) + var workingDir, err = cmd.DeriveLocalSyncPath(cmd.Config, dir) if err != nil { - return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) + return "", "", err } // Determine the volume name to be used across all operating systems. // For cross-compatibility the way this volume is set up will vary. volumeName := cmd.GetVolumeName(cmd.Config, workingDir) - switch platform := runtime.GOOS; platform { - case "linux": - cmd.out.Verbose("Setting up local volume: %s", volumeName) - return cmd.SetupBindVolume(volumeName, workingDir) - default: - cmd.out.Verbose("Starting sync with volume: %s", volumeName) - return cmd.StartUnisonSync(ctx, volumeName, cmd.Config, workingDir) - } + return volumeName, workingDir, nil } // StartUnisonSync will create and launch the volumes and containers on systems that need/support Unison @@ -132,9 +323,9 @@ func (cmd *ProjectSync) StartUnisonSync(ctx *cli.Context, volumeName string, con } cmd.out.Info("Sync volume '%s' created", volumeName) cmd.out.SpinWithVerbose(fmt.Sprintf("Starting sync container: %s (same name)", volumeName)) - unisonMinorVersion := cmd.GetUnisonMinorVersion() + unisonMinorVersion := util.GetUnisonMinorVersion() - cmd.out.Verbose("Local Unison version for compatibilty: %s", unisonMinorVersion) + cmd.out.Verbose("Local Unison version for compatibility: %s", unisonMinorVersion) util.Command("docker", "container", "stop", volumeName).Run() containerArgs := []string{ "container", "run", "--detach", "--rm", @@ -157,12 +348,12 @@ func (cmd *ProjectSync) StartUnisonSync(ctx *cli.Context, volumeName string, con cmd.out.SpinWithVerbose("Initializing file sync...") // Determine the location of the local Unison log file. - var logFile = fmt.Sprintf("%s.log", volumeName) + var logFile = cmd.LogFileName(volumeName) // Remove the log file, the existence of the log file will mean that sync is // up and running. If the logfile does not exist, do not complain. If the // filesystem cannot delete the file when it exists, it will lead to errors. - if err := util.RemoveFile(logFile, workingDir); err != nil { - cmd.out.Verbose("Could not remove Unison log file: %s: %s", logFile, err.Error()) + if removeErr := util.RemoveFile(logFile, workingDir); removeErr != nil { + cmd.out.Verbose("Could not remove Unison log file: %s: %s", logFile, removeErr.Error()) } // Initiate local Unison process. @@ -219,31 +410,11 @@ func (cmd *ProjectSync) SetupBindVolume(volumeName string, workingDir string) er return cmd.Success("Bind volume created") } -// RunStop executes the `rig project sync:stop` command to shut down and unison containers -func (cmd *ProjectSync) RunStop(ctx *cli.Context) error { - if runtime.GOOS == "linux" { - return cmd.Success("No Unison container to stop, using local bind volume") - } - cmd.out.Spin(fmt.Sprintf("Stopping Unison container")) - - cmd.Config = NewProjectConfig() - if cmd.Config.NotEmpty() { - cmd.out.Verbose("Loaded project configuration from %s", cmd.Config.Path) - } - - // Determine the working directory for CWD-sensitive operations. - var workingDir, err = cmd.DeriveLocalSyncPath(cmd.Config, ctx.String("dir")) - if err != nil { - return cmd.Failure(err.Error(), "SYNC-PATH-ERROR", 12) - } - - volumeName := cmd.GetVolumeName(cmd.Config, workingDir) - cmd.out.Spin(fmt.Sprintf("Stopping Unison container (%s)", volumeName)) - if err := util.Command("docker", "container", "stop", volumeName).Run(); err != nil { - return cmd.Failure(err.Error(), "SYNC-CONTAINER-FAILURE", 13) - } - - return cmd.Success(fmt.Sprintf("Unison container '%s' stopped", volumeName)) +// LogFileName gets the unison sync file name. +// Be sure to convert it to an absolute path if used with functions that cannot +// use the working directory context. +func (cmd *ProjectSync) LogFileName(name string) string { + return fmt.Sprintf("%s.log", name) } // GetVolumeName will find the volume name through a variety of fall backs @@ -302,9 +473,11 @@ func (cmd *ProjectSync) WaitForUnisonContainer(containerName string, timeoutSeco cmd.out.Verbose("Checking for Unison network connection on %s %d", ip, unisonPort) for i := 1; i <= timeoutLoops; i++ { + cmd.out.Verbose("Attempt #%d...", i) conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", ip, unisonPort)) if err == nil { conn.Close() + cmd.out.Verbose("Connected to unison on %s", containerName) return ip, nil } @@ -312,7 +485,7 @@ func (cmd *ProjectSync) WaitForUnisonContainer(containerName string, timeoutSeco time.Sleep(timeoutLoopSleep) } - return "", fmt.Errorf("sync container %s failed to start", containerName) + return "", fmt.Errorf("sync container %s is unreachable by unison", containerName) } // WaitForSyncInit will wait for the local unison process to finish initializing @@ -329,19 +502,20 @@ func (cmd *ProjectSync) WaitForSyncInit(logFile string, workingDir string, timeo var tempFile = ".rig-check-sync-start" if err := util.TouchFile(tempFile, workingDir); err != nil { - cmd.out.Channel.Error.Fatal("Could not create file used to detect initial sync: %s", err.Error()) + cmd.out.Channel.Error.Fatal(fmt.Sprintf("Could not create file used to detect initial sync: %s", err.Error())) } cmd.out.Verbose("Creating temporary file so we can watch for Unison initialization: %s", tempFile) var timeoutLoopSleep = time.Duration(100) * time.Millisecond // * 10 here because we loop once every 100 ms and we want to get to seconds var timeoutLoops = timeoutSeconds * 10 + var statSleep = time.Duration(syncWaitSeconds) * time.Second for i := 1; i <= timeoutLoops; i++ { + cmd.out.Verbose("Checking that a file can sync: Attempt #%d", i) statInfo, err := os.Stat(logFilePath) if err == nil { cmd.out.Info("Initial sync detected") - cmd.out.SpinWithVerbose("Waiting for initial sync to finish") // Initialize at -2 to force at least one loop var lastSize = int64(-2) @@ -379,17 +553,6 @@ func (cmd *ProjectSync) WaitForSyncInit(logFile string, workingDir string, timeo return fmt.Errorf("Failed to detect start of initial sync") } -// GetUnisonMinorVersion will return the local Unison version to try to load a compatible unison image -// This function discovers a semver like 2.48.4 and return 2.48 -func (cmd *ProjectSync) GetUnisonMinorVersion() string { - output, _ := util.Command("unison", "-version").Output() - re := regexp.MustCompile(`unison version (\d+\.\d+\.\d+)`) - rawVersion := re.FindAllStringSubmatch(string(output), -1)[0][1] - v := version.Must(version.NewVersion(rawVersion)) - segments := v.Segments() - return fmt.Sprintf("%d.%d", segments[0], segments[1]) -} - // DeriveLocalSyncPath will derive the source path for the local host side of the file sync. // If there is no override, use an empty string. func (cmd *ProjectSync) DeriveLocalSyncPath(config *ProjectConfig, override string) (string, error) { diff --git a/util/docker.go b/util/docker.go index 396124c..8494d19 100644 --- a/util/docker.go +++ b/util/docker.go @@ -1,6 +1,8 @@ package util import ( + "fmt" + "os/exec" "regexp" "strings" "time" @@ -49,6 +51,19 @@ func GetDockerServerMinAPIVersion() (*version.Version, error) { return version.Must(version.NewVersion(strings.TrimSpace(string(output)))), nil } +// ContainerRunning determines if the named container is live. +func ContainerRunning(name string) bool { + filter := fmt.Sprintf("name=^/%s", name) + if out, err := Command("docker", "ps", "-aq", "--filter", filter).Output(); err == nil { + id := strings.TrimSpace(string(out)) + /* #nosec */ + _, code, err := CaptureCommand(exec.Command("docker", "top", id)) + return code == 0 && err == nil + } + + return false +} + // ImageOlderThan determines the age of the Docker Image and whether the image is older than the designated timestamp. func ImageOlderThan(image string, elapsedSeconds float64) (bool, float64, error) { output, err := Command("docker", "inspect", "--format", "{{.Created}}", image).Output() diff --git a/util/filesystem.go b/util/filesystem.go index 28c16af..5bc4b2f 100644 --- a/util/filesystem.go +++ b/util/filesystem.go @@ -14,7 +14,10 @@ func GetExecutableDir() (string, error) { } // AbsJoin joins the two path segments, ensuring they form an absolute path. -func AbsJoin(baseDir string, suffixPath string) (string, error) { +func AbsJoin(baseDir, suffixPath string) (string, error) { + if len(baseDir) == 0 { + baseDir = string(filepath.Separator) + } absoluteBaseDir, err := filepath.Abs(baseDir) if err != nil { return "", fmt.Errorf("Unrecognized working directory: %s: %s", baseDir, err.Error()) @@ -23,8 +26,19 @@ func AbsJoin(baseDir string, suffixPath string) (string, error) { return filepath.Join(absoluteBaseDir, suffixPath), nil } +// FileExists reports whether a file exists. +func FileExists(pathToFile, workingDir string) bool { + absoluteFilePath, err := AbsJoin(workingDir, pathToFile) + if err == nil { + _, statErr := os.Stat(absoluteFilePath) + return statErr == nil + } + + return false +} + // RemoveFile removes the designated file relative to the Working Directory. -func RemoveFile(pathToFile string, workingDir string) error { +func RemoveFile(pathToFile, workingDir string) error { absoluteFilePath, err := AbsJoin(workingDir, pathToFile) if err != nil { return err @@ -33,6 +47,32 @@ func RemoveFile(pathToFile string, workingDir string) error { return os.Remove(absoluteFilePath) } +// RemoveFileGlob removes all files under the working directory that match the glob. +// This recursively traverses all sub-directories. If a logger is passed the action +// will be verbosely logged, otherwise pass nil to skip all output. +func RemoveFileGlob(glob, targetDirectory string, logger *RigLogger) error { + return filepath.Walk(targetDirectory, func(path string, info os.FileInfo, err error) error { + if err == nil && info.IsDir() { + globPath := filepath.Join(path, glob) + if files, globErr := filepath.Glob(globPath); globErr == nil { + for _, file := range files { + if logger != nil { + logger.Verbose("Removing file '%s'...", file) + } + if removeErr := RemoveFile(file, ""); removeErr != nil { + logger.Verbose("Remove error '%s'", removeErr) + return removeErr + } + } + } else { + return globErr + } + } + + return nil + }) +} + // TouchFile creates an empty file, usually for temporary use. // @see https://stackoverflow.com/questions/35558787/create-an-empty-text-file/35558965 func TouchFile(pathToFile string, workingDir string) error { diff --git a/util/unison.go b/util/unison.go new file mode 100644 index 0000000..693de6b --- /dev/null +++ b/util/unison.go @@ -0,0 +1,19 @@ +package util + +import ( + "fmt" + "regexp" + + "github.com/hashicorp/go-version" +) + +// GetUnisonMinorVersion will return the local Unison version to try to load a compatible unison image +// This function discovers a semver like 2.48.4 and return 2.48 +func GetUnisonMinorVersion() string { + output, _ := Command("unison", "-version").Output() + re := regexp.MustCompile(`unison version (\d+\.\d+\.\d+)`) + rawVersion := re.FindAllStringSubmatch(string(output), -1)[0][1] + v := version.Must(version.NewVersion(rawVersion)) + segments := v.Segments() + return fmt.Sprintf("%d.%d", segments[0], segments[1]) +}