From 52385495641d9f06d2d4f0dd2b77b659f938b77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 16:27:21 +0200 Subject: [PATCH 01/28] [FEATURE] Sync / watch strategy unison with unox --- lib/docker-sync/preconditions.rb | 6 + lib/docker-sync/sync_process.rb | 6 + lib/docker-sync/sync_strategy/unison-unox.rb | 191 +++++++++++++++++++ lib/docker-sync/watch_strategy/unison.rb | 45 +++++ 4 files changed, 248 insertions(+) create mode 100644 lib/docker-sync/sync_strategy/unison-unox.rb create mode 100644 lib/docker-sync/watch_strategy/unison.rb diff --git a/lib/docker-sync/preconditions.rb b/lib/docker-sync/preconditions.rb index 1fe19d3b..b6e7e574 100644 --- a/lib/docker-sync/preconditions.rb +++ b/lib/docker-sync/preconditions.rb @@ -43,4 +43,10 @@ def self.unison_available raise('Could not find unison binary in path. Please install it, e.g. using "brew install unison"') end end + + def self.unox_available + if (find_executable0 'unison-fsmonitor').nil? + raise('Could not find unison-fsmonitor binary in path. Please install it, see https://github.com/hnsl/unox') + end + end end \ No newline at end of file diff --git a/lib/docker-sync/sync_process.rb b/lib/docker-sync/sync_process.rb index fac01b81..71a6717a 100644 --- a/lib/docker-sync/sync_process.rb +++ b/lib/docker-sync/sync_process.rb @@ -3,9 +3,11 @@ require 'docker-sync/sync_strategy/rsync' require 'docker-sync/sync_strategy/unison' require 'docker-sync/sync_strategy/unison-dualside' +require 'docker-sync/sync_strategy/unison-unox' # noinspection RubyResolve require 'docker-sync/watch_strategy/fswatch' require 'docker-sync/watch_strategy/dummy' +require 'docker-sync/watch_strategy/unison' module Docker_Sync class SyncProcess @@ -39,6 +41,8 @@ def set_sync_strategy @sync_strategy = Docker_Sync::SyncStrategy::Unison.new(@sync_name, @options) when 'unison-dualside' @sync_strategy = Docker_Sync::SyncStrategy::Unison_DualSide.new(@sync_name, @options) + when 'unison-unox' + @sync_strategy = Docker_Sync::SyncStrategy::Unison_Unox.new(@sync_name, @options) else @sync_strategy = Docker_Sync::SyncStrategy::Rsync.new(@sync_name, @options) end @@ -54,6 +58,8 @@ def set_watch_strategy @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) when 'disable','dummy' @watch_strategy = Docker_Sync::WatchStrategy::Dummy.new(@sync_name, @options) + when 'unison' + @watch_strategy = Docker_Sync::WatchStrategy::Unison.new(@sync_name, @options) else @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) end diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb new file mode 100644 index 00000000..cdcb5a8b --- /dev/null +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -0,0 +1,191 @@ +require 'thor/shell' +require 'docker-sync/preconditions' +require 'docker-sync/execution' +require 'open3' +require 'socket' + +module Docker_Sync + module SyncStrategy + class Unison_Unox + include Thor::Shell + include Execution + + @options + @sync_name + @watch_thread + @local_server_pid + UNISON_CONTAINER_PORT = '5000' + def initialize(sync_name, options) + @options = options + @sync_name = @options['project'] != '' ? @options['project'] + '_' + sync_name : sync_name + # if a custom image is set, apply it + if @options.key?('image') + @docker_image = @options['image'] + else + @docker_image = 'eugenmayer/unison:unox' + end + begin + Preconditions::unison_available + Preconditions::unox_available + rescue Exception => e + say_status 'error', "#{@sync_name} has been configured to sync with unison, but no unison available", :red + say_status 'error', e.message, :red + exit 1 + end + end + + def run + start_container + sync + end + + def watch + args = sync_options + args.push("-repeat watch") + cmd = 'unison ' + args.join(' ') + + say_status 'command', cmd, :white if @options['verbose'] + forkexec(cmd, "Sync #{@sync_name}", :blue) + end + + def sync + args = sync_options + cmd = 'unison ' + args.join(' ') + + say_status 'command', cmd, :white if @options['verbose'] + + stdout, stderr, exit_status = Open3.capture3(cmd) + if not exit_status.success? + say_status 'error', "Error starting sync, exit code #{$?.exitstatus}", :red + say_status 'message', stderr + else + say_status 'ok', "Synced #{@options['src']}", :white + if @options['verbose'] + say_status 'output', stdout + end + end + end + + def sync_options + args = [] + + unless @options['sync_excludes'].nil? + args = @options['sync_excludes'].map { |pattern| "-ignore='Path #{pattern}'" } + args + end + args.push(@options['src']) + args.push('-auto') + args.push('-batch') + args.push('-owner') if @options['sync_userid'] == 'from_host' + args.push('-numericids') if @options['sync_userid'] == 'from_host' + args.push(@options['sync_args']) if @options.key?('sync_args') + args.push("socket://#{@options['sync_host_ip']}:#{@options['sync_host_port']}/") + #args.push('-debug all') if @options['verbose'] + + if @options.key?('sync_user') || @options.key?('sync_group') || @options.key?('sync_groupid') + raise('Unison does not support sync_user, sync_group, sync_groupid - please use rsync if you need that') + end + if @options.key?('sync_userid') && @options['sync_userid'] != 'from_host' + raise('Unison does not support sync_userid with a parameter different than \'from_host\'') + end + return args + end + + def start_container + say_status 'ok', 'Starting unison', :white + container_name = get_container_name + volume_name = get_volume_name + env = {} + + env['ENABLE_WATCH'] = 'true' + # Excludes for fswatch and unison uses the same value both in the host and in the container + env['FSWATCH_EXCLUDES'] = @options['watch_excludes'].map { |pattern| "--exclude='#{pattern}'" }.join(' ') if @options.key?('watch_excludes') + env['UNISON_EXCLUDES'] = @options['sync_excludes'].map { |pattern| "-ignore='Path #{pattern}'" }.join(' ') if @options.key?('sync_excludes') + # Specify the IP of the host to call the host unison server from the container + env['HOST_IP'] = get_host_ip + env['UNISON_HOST_PORT'] = @options['unison-dualside_host_server_port'] + + if @options['sync_userid'] == 'from_host' + env['UNISON_OWNER_UID'] = Process.uid + end + + additional_docker_env = env.map{ |key,value| "-e #{key}=\"#{value}\"" }.join(' ') + running = `docker ps --filter 'status=running' --filter 'name=#{container_name}' | grep #{container_name}` + if running == '' + say_status 'ok', "#{container_name} container not running", :white if @options['verbose'] + exists = `docker ps --filter "status=exited" --filter "name=#{container_name}" | grep #{container_name}` + if exists == '' + say_status 'ok', "creating #{container_name} container", :white if @options['verbose'] + cmd = "docker run -p '#{@options['sync_host_port']}:#{UNISON_CONTAINER_PORT}' \ + -v #{volume_name}:#{@options['dest']} \ + -e UNISON_DIR=#{@options['dest']} \ + #{additional_docker_env} \ + --name #{container_name} \ + -d #{@docker_image}" + else + say_status 'ok', "starting #{container_name} container", :white if @options['verbose'] + cmd = "docker start #{container_name}" + end + say_status 'command', cmd, :white if @options['verbose'] + `#{cmd}` || raise('Start failed') + else + say_status 'ok', "#{container_name} container still running", :blue + end + say_status 'ok', "starting initial sync of #{container_name}", :white if @options['verbose'] + # this sleep is needed since the container could be not started + sleep 5 # TODO: replace with unison -testserver + sync + say_status 'success', 'Unison server started', :green + end + + # Try to guess the IP of the host to call the unison server from the container + # Or use the value specified in the config option + def get_host_ip + if @options['unison-dualside_host_ip'] != 'auto' && @options.key?('unison-dualside_host_ip') + host_ip = @options['host_ip'] + elsif @options['unison-dualside_host_ip'] == 'auto' || (not @options.key?('unison-dualside_host_ip')) + host_ip = Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }.ip_address + end + return host_ip + end + + # Kill the local unison server + def stop_local_server + Process.kill "TERM", @local_server_pid + Process.wait @local_server_pid + end + + def get_container_name + return "#{@sync_name}" + end + + def get_volume_name + return @sync_name + end + + def stop_container + `docker stop #{get_container_name}` + end + + def reset_container + stop_container + `docker rm #{get_container_name}` + `docker volume rm #{get_volume_name}` + end + + def clean + reset_container + end + + def stop + say_status 'ok', "Stopping sync container #{get_container_name}" + begin + stop_container + stop_local_server + rescue Exception => e + say_status 'error', "Stopping failed of #{get_container_name}:", :red + puts e.message + end + end + end + end +end diff --git a/lib/docker-sync/watch_strategy/unison.rb b/lib/docker-sync/watch_strategy/unison.rb new file mode 100644 index 00000000..0d2bf10a --- /dev/null +++ b/lib/docker-sync/watch_strategy/unison.rb @@ -0,0 +1,45 @@ +require 'thor/shell' +require 'docker-sync/execution' +require 'docker-sync/preconditions' +require 'docker-sync/sync_strategy/unison-unox' + +module Docker_Sync + module WatchStrategy + class Unison + include Execution + + @options + @sync_name + @watch_fork + + def initialize(sync_name, options) + @options = options + @sync_name = sync_name + @watch_fork = nil + @unison = Docker_Sync::SyncStrategy::Unison_Unox.new(@sync_name, @options) + end + + def run + @watch_fork = @unison.watch + end + + def stop + Process.kill "TERM", @watch_fork + Process.wait @watch_fork + end + + def clean + end + + def watch + end + + def watch_options + end + + def watch_thread + return nil + end + end + end +end From 44f1c160e04a8dd90a0129df9af1908c7f9c841c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 16:30:29 +0200 Subject: [PATCH 02/28] [BUGFIX] Unox syncing doesn't work in docker-sync - PB: while the command launched has no errors, the sync/watch doesn't work. - FIX: it looks like that creating a new process instead of a thread resolves the issue. - TODO: ensure that this is the adequate way to resolve the issue --- lib/docker-sync/execution.rb | 43 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/docker-sync/execution.rb b/lib/docker-sync/execution.rb index 1ad89610..7785883a 100644 --- a/lib/docker-sync/execution.rb +++ b/lib/docker-sync/execution.rb @@ -5,6 +5,22 @@ module Execution Thread.abort_on_exception = true + def exec(command, prefix, color) + Open3.popen3(command) do |_, stdout, stderr, _| + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_out = stdout.gets + say_status prefix, line_out, color + end + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_err = stderr.gets + say_status prefix, line_err, :red + end + + end + end + def threadexec(command, prefix = nil, color = nil) if prefix.nil? @@ -17,20 +33,25 @@ def threadexec(command, prefix = nil, color = nil) end Thread.new { - Open3.popen3(command) do |_, stdout, stderr, _| + exec(command, prefix, color) + } - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_out = stdout.gets - say_status prefix, line_out, color - end + end - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_err = stderr.gets - say_status prefix, line_err, :red - end + def forkexec(command, prefix = nil, color = nil) - end - } + if prefix.nil? + # TODO: probably pick the command name without args + prefix = 'unknown' + end + + if color.nil? + color = :cyan + end + + Process.fork do + exec(command, prefix, color) + end end From e0d2c7e124ac4489737cf0d008e6dec65591ab2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 16:44:57 +0200 Subject: [PATCH 03/28] [BUGFIX] Unison processes are not killed - PB: docker-sync exit because of the lack of threads. Indeed, unison is launched as fork. So forked unison processes are not killed when docker-sync ends. - FIX: sleep indefinitely if there are no threads --- lib/docker-sync/sync_manager.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index efa0291e..16c5ae5f 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -121,10 +121,16 @@ def run(sync_name = nil) def join_threads begin + has_threads = false @sync_processes.each do |sync_process| - sync_process.watch_thread.join unless sync_process.watch_thread.nil? + if sync_process.watch_thread + sync_process.watch_thread.join + has_threads = true + end end + sleep unless has_threads + rescue SystemExit, Interrupt say_status 'shutdown', 'Shutting down...', :blue @sync_processes.each do |sync_process| From 979914b44f10973eefcf7515f79f2f78ba03026b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 16:45:47 +0200 Subject: [PATCH 04/28] [BUGFIX] Remove code from previous PR --- lib/docker-sync/sync_strategy/unison-unox.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index cdcb5a8b..151203a8 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -17,7 +17,7 @@ class Unison_Unox UNISON_CONTAINER_PORT = '5000' def initialize(sync_name, options) @options = options - @sync_name = @options['project'] != '' ? @options['project'] + '_' + sync_name : sync_name + @sync_name = sync_name # if a custom image is set, apply it if @options.key?('image') @docker_image = @options['image'] From 21c7925ef5ab616e603b2bfdb504bea4f04e5fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 18:20:28 +0200 Subject: [PATCH 05/28] [TASK] Cleanup, remove unused code --- lib/docker-sync/sync_strategy/unison-unox.rb | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 151203a8..56143f0b 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -96,13 +96,7 @@ def start_container volume_name = get_volume_name env = {} - env['ENABLE_WATCH'] = 'true' - # Excludes for fswatch and unison uses the same value both in the host and in the container - env['FSWATCH_EXCLUDES'] = @options['watch_excludes'].map { |pattern| "--exclude='#{pattern}'" }.join(' ') if @options.key?('watch_excludes') env['UNISON_EXCLUDES'] = @options['sync_excludes'].map { |pattern| "-ignore='Path #{pattern}'" }.join(' ') if @options.key?('sync_excludes') - # Specify the IP of the host to call the host unison server from the container - env['HOST_IP'] = get_host_ip - env['UNISON_HOST_PORT'] = @options['unison-dualside_host_server_port'] if @options['sync_userid'] == 'from_host' env['UNISON_OWNER_UID'] = Process.uid @@ -137,17 +131,6 @@ def start_container say_status 'success', 'Unison server started', :green end - # Try to guess the IP of the host to call the unison server from the container - # Or use the value specified in the config option - def get_host_ip - if @options['unison-dualside_host_ip'] != 'auto' && @options.key?('unison-dualside_host_ip') - host_ip = @options['host_ip'] - elsif @options['unison-dualside_host_ip'] == 'auto' || (not @options.key?('unison-dualside_host_ip')) - host_ip = Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }.ip_address - end - return host_ip - end - # Kill the local unison server def stop_local_server Process.kill "TERM", @local_server_pid From 7987b246c72ca77a05a059a23759633484693971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 18:20:55 +0200 Subject: [PATCH 06/28] [FEATURE] Add support for user/uid mapping --- lib/docker-sync/sync_strategy/unison-unox.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 56143f0b..70c24073 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -75,18 +75,12 @@ def sync_options args.push(@options['src']) args.push('-auto') args.push('-batch') - args.push('-owner') if @options['sync_userid'] == 'from_host' - args.push('-numericids') if @options['sync_userid'] == 'from_host' args.push(@options['sync_args']) if @options.key?('sync_args') args.push("socket://#{@options['sync_host_ip']}:#{@options['sync_host_port']}/") - #args.push('-debug all') if @options['verbose'] - if @options.key?('sync_user') || @options.key?('sync_group') || @options.key?('sync_groupid') + if @options.key?('sync_group') || @options.key?('sync_groupid') raise('Unison does not support sync_user, sync_group, sync_groupid - please use rsync if you need that') end - if @options.key?('sync_userid') && @options['sync_userid'] != 'from_host' - raise('Unison does not support sync_userid with a parameter different than \'from_host\'') - end return args end @@ -97,9 +91,11 @@ def start_container env = {} env['UNISON_EXCLUDES'] = @options['sync_excludes'].map { |pattern| "-ignore='Path #{pattern}'" }.join(' ') if @options.key?('sync_excludes') - + env['UNISON_OWNER'] = @options['sync_user'] if @options.key?('sync_user') if @options['sync_userid'] == 'from_host' env['UNISON_OWNER_UID'] = Process.uid + else + env['UNISON_OWNER_UID'] = @options['sync_userid'] if @options.key?('sync_userid') end additional_docker_env = env.map{ |key,value| "-e #{key}=\"#{value}\"" }.join(' ') From a22ff518f949a63535673143aec8e703a52ebc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 18:44:33 +0200 Subject: [PATCH 07/28] [FEATURE] Prevent error if uid is already in use --- example/create-data/entrypoint.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/example/create-data/entrypoint.sh b/example/create-data/entrypoint.sh index a7bef735..6a8a2ab2 100644 --- a/example/create-data/entrypoint.sh +++ b/example/create-data/entrypoint.sh @@ -1,12 +1,19 @@ #!/bin/sh -echo "$1" > /command.txt +echo "$1" if [ "$1" == "watch" ]; then - adduser -u $(stat -c '%u' /data) -D www-data + run_with_user=www-data + + # prevent error if uid is already in use + if ! cut -d: -f3 /etc/passwd | grep -q $(stat -c '%u' /data); then + adduser -u $(stat -c '%u' /data) -D www-data + else + $run_with_user=$(awk -F: "/:$UNISON_OWNER_UID:/{print \$1}" /etc/passwd) + fi while true; do - su www-data -c 'date >> /data/filefromcontainer.txt'; + su $run_with_userr -c 'date >> /data/filefromcontainer.txt'; sleep 5; done fi From e3bfc114fc45475e5331107b309a6382dac7c4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Thu, 21 Jul 2016 18:45:02 +0200 Subject: [PATCH 08/28] [TASK] Add example for unison-unox --- example/data4/filefromhost.txt | 1 + example/docker-compose-dev.yml | 11 ++++++++--- example/docker-compose.yml | 10 +++++++--- example/docker-sync.yml | 27 +++++++++++++++++++-------- 4 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 example/data4/filefromhost.txt diff --git a/example/data4/filefromhost.txt b/example/data4/filefromhost.txt new file mode 100644 index 00000000..df3f5807 --- /dev/null +++ b/example/data4/filefromhost.txt @@ -0,0 +1 @@ +File from host \ No newline at end of file diff --git a/example/docker-compose-dev.yml b/example/docker-compose-dev.yml index 06addbdf..0cc30f5a 100644 --- a/example/docker-compose-dev.yml +++ b/example/docker-compose-dev.yml @@ -9,14 +9,19 @@ services: otherapp: volumes_from: - container:simpleexample-sync:rw # will be mounted on /app/code - appthatproducedata: +# appthatproducedata: +# volumes_from: +# - container:unison-dualside-sync:rw # will be mounted on /data + newapp: volumes_from: - - container:unison-dualside-sync:rw # will be mounted on /data + - container:unison-unox:rw # will be mounted on /data volumes: fullexample-sync: external: true simpleexample-sync: external: true - unison-dualside-sync: +# unison-dualside-sync: +# external: true + unison-unox: external: true \ No newline at end of file diff --git a/example/docker-compose.yml b/example/docker-compose.yml index e9e69c7e..64fe300d 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -10,7 +10,11 @@ services: image: alpine container_name: 'simpleexample_app' command: ['watch', '-n1', 'cat /app/code/somefile.txt'] - appthatproducedata: +# appthatproducedata: +# build: ./create-data +# container_name: 'unison-dualside_app' +# command: ['watch', '-n1', 'cat /data/filefromhost.txt'] + newapp: build: ./create-data - container_name: 'unison-dualside_app' - command: ['watch', '-n1', 'cat /data/filefromhost.txt'] + container_name: 'unison-unox_app' + command: ['watch', '-n1', 'cat /data/filefromhost.txt'] \ No newline at end of file diff --git a/example/docker-sync.yml b/example/docker-sync.yml index 48755e60..5f6ad9c6 100644 --- a/example/docker-sync.yml +++ b/example/docker-sync.yml @@ -68,13 +68,24 @@ syncs: sync_host_ip: 'localhost' sync_host_port: 20872 sync_strategy: 'unison' # this time we pick unison - unison-dualside-sync: - src: './data3/' +# unison-dualside-sync: +# src: './data3/' +# dest: '/data' +# sync_host_ip: '172.16.210.131' +# sync_host_port: 20873 +# sync_userid: 'from_host' +# unison-dualside_host_ip: 'auto' +# unison-dualside_host_server_port: 6000 +# sync_strategy: 'unison-dualside' + unison-unox: + src: './data4/' dest: '/data' - image: 'mickaelperrin/docker-unison-fsevents' - sync_host_ip: '172.16.210.131' + image: 'mickaelperrin/docker-unison-unox' + sync_host_ip: 'localhost' sync_host_port: 20873 - sync_userid: 'from_host' - unison-dualside_host_ip: 'auto' - unison-dualside_host_server_port: 6000 - sync_strategy: 'unison-dualside' + sync_strategy: 'unison-unox' + watch_strategy: 'unison' + sync_user: 'www-data' + sync_userid: '33' + + From ee9ff465bd610378d05d29785f24c49a9ee91582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Fri, 22 Jul 2016 00:10:04 +0200 Subject: [PATCH 09/28] [FEATURE] Host port discovery --- example/docker-sync.yml | 1 - lib/docker-sync/sync_manager.rb | 4 +++- lib/docker-sync/sync_strategy/unison-unox.rb | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/example/docker-sync.yml b/example/docker-sync.yml index 5f6ad9c6..743027c2 100644 --- a/example/docker-sync.yml +++ b/example/docker-sync.yml @@ -82,7 +82,6 @@ syncs: dest: '/data' image: 'mickaelperrin/docker-unison-unox' sync_host_ip: 'localhost' - sync_host_port: 20873 sync_strategy: 'unison-unox' watch_strategy: 'unison' sync_user: 'www-data' diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index 16c5ae5f..bc3aa5c5 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -78,7 +78,9 @@ def validate_config(config) end def validate_sync_config(name, sync_config) - %w[src dest sync_host_port].each do |key| + config_mandatory = %w[src dest] + config_mandatory.push('sync_host_port') unless name == 'unison-unox' #TODO: Implement autodisovery for other strategies + %w[src dest].each do |key| raise ("#{name} does not have #{key} condiguration value set - this is mandatory") unless sync_config.key?(key) end end diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 70c24073..9ceb8172 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -76,7 +76,8 @@ def sync_options args.push('-auto') args.push('-batch') args.push(@options['sync_args']) if @options.key?('sync_args') - args.push("socket://#{@options['sync_host_ip']}:#{@options['sync_host_port']}/") + sync_host_port = get_host_port(get_container_name, UNISON_CONTAINER_PORT) + args.push("socket://#{@options['sync_host_ip']}:#{sync_host_port}") if @options.key?('sync_group') || @options.key?('sync_groupid') raise('Unison does not support sync_user, sync_group, sync_groupid - please use rsync if you need that') @@ -105,7 +106,7 @@ def start_container exists = `docker ps --filter "status=exited" --filter "name=#{container_name}" | grep #{container_name}` if exists == '' say_status 'ok', "creating #{container_name} container", :white if @options['verbose'] - cmd = "docker run -p '#{@options['sync_host_port']}:#{UNISON_CONTAINER_PORT}' \ + cmd = "docker run -p '127.0.0.1::#{UNISON_CONTAINER_PORT}' \ -v #{volume_name}:#{@options['dest']} \ -e UNISON_DIR=#{@options['dest']} \ #{additional_docker_env} \ @@ -127,6 +128,18 @@ def start_container say_status 'success', 'Unison server started', :green end + def get_host_port(container_name, container_port) + cmd = 'docker inspect --format=" {{ .NetworkSettings.Ports }} " ' + container_name + ' | sed -r "s/.*map\[' + container_port + '[^ ]+\s([0-9]+).*/\1/"' + say_status 'command', cmd, :white if @options['verbose'] + stdout, stderr, exit_status = Open3.capture3(cmd) + if not exit_status.success? + say_status 'command', cmd + say_status 'error', "Error getting mapped port, exit code #{$?.exitstatus}", :red + say_status 'message', stderr + end + return stdout.gsub("\n",'') + end + # Kill the local unison server def stop_local_server Process.kill "TERM", @local_server_pid From 4910928c0f8dedb853cbc2142fa7854315a923aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Fri, 22 Jul 2016 00:32:01 +0200 Subject: [PATCH 10/28] [FEATURE] Set unison as default watcher for unox --- example/docker-sync.yml | 1 - lib/docker-sync/sync_process.rb | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/example/docker-sync.yml b/example/docker-sync.yml index 743027c2..b7148e42 100644 --- a/example/docker-sync.yml +++ b/example/docker-sync.yml @@ -83,7 +83,6 @@ syncs: image: 'mickaelperrin/docker-unison-unox' sync_host_ip: 'localhost' sync_strategy: 'unison-unox' - watch_strategy: 'unison' sync_user: 'www-data' sync_userid: '33' diff --git a/lib/docker-sync/sync_process.rb b/lib/docker-sync/sync_process.rb index 71a6717a..e4a54e0c 100644 --- a/lib/docker-sync/sync_process.rb +++ b/lib/docker-sync/sync_process.rb @@ -64,7 +64,11 @@ def set_watch_strategy @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) end else - @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) + if @options['sync_strategy'] == 'unison-unox' + @watch_strategy = Docker_Sync::WatchStrategy::Unison.new(@sync_name, @options) + else + @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) + end end end From a28ec76bfab50adc5de1bc297d5932de2ec46a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Fri, 22 Jul 2016 00:33:09 +0200 Subject: [PATCH 11/28] [BUGFIX] sync_host_port is no more checked --- lib/docker-sync/sync_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index bc3aa5c5..47637a73 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -80,7 +80,7 @@ def validate_config(config) def validate_sync_config(name, sync_config) config_mandatory = %w[src dest] config_mandatory.push('sync_host_port') unless name == 'unison-unox' #TODO: Implement autodisovery for other strategies - %w[src dest].each do |key| + config_mandatory.each do |key| raise ("#{name} does not have #{key} condiguration value set - this is mandatory") unless sync_config.key?(key) end end From 2ca57156e329982fd5a3ba10db0a933500f5983a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Fri, 22 Jul 2016 00:33:09 +0200 Subject: [PATCH 12/28] [BUGFIX] sync_host_port is no more checked --- lib/docker-sync/sync_manager.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index bc3aa5c5..367d5ac6 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -79,8 +79,8 @@ def validate_config(config) def validate_sync_config(name, sync_config) config_mandatory = %w[src dest] - config_mandatory.push('sync_host_port') unless name == 'unison-unox' #TODO: Implement autodisovery for other strategies - %w[src dest].each do |key| + config_mandatory.push('sync_host_port') unless sync_config['sync_strategy'] == 'unison-unox' #TODO: Implement autodisovery for other strategies + config_mandatory.each do |key| raise ("#{name} does not have #{key} condiguration value set - this is mandatory") unless sync_config.key?(key) end end From 4d2c7fd811f5c983a233e07c4d0c00fc03e37a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Fri, 22 Jul 2016 01:15:02 +0200 Subject: [PATCH 13/28] [TASK] Remove old code --- lib/docker-sync/sync_strategy/unison-unox.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 9ceb8172..5c9f0c37 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -140,12 +140,6 @@ def get_host_port(container_name, container_port) return stdout.gsub("\n",'') end - # Kill the local unison server - def stop_local_server - Process.kill "TERM", @local_server_pid - Process.wait @local_server_pid - end - def get_container_name return "#{@sync_name}" end @@ -172,7 +166,6 @@ def stop say_status 'ok', "Stopping sync container #{get_container_name}" begin stop_container - stop_local_server rescue Exception => e say_status 'error', "Stopping failed of #{get_container_name}:", :red puts e.message From 4820f16592ce00d487468eb5b1d2458b4aad5833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 25 Jul 2016 17:09:08 +0200 Subject: [PATCH 14/28] [TASK] Remove factorization in execution.rb see https://github.com/EugenMayer/docker-sync/pull/64#issuecomment-234721905 --- lib/docker-sync/execution.rb | 48 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/lib/docker-sync/execution.rb b/lib/docker-sync/execution.rb index 7785883a..15588788 100644 --- a/lib/docker-sync/execution.rb +++ b/lib/docker-sync/execution.rb @@ -5,22 +5,6 @@ module Execution Thread.abort_on_exception = true - def exec(command, prefix, color) - Open3.popen3(command) do |_, stdout, stderr, _| - - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_out = stdout.gets - say_status prefix, line_out, color - end - - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_err = stderr.gets - say_status prefix, line_err, :red - end - - end - end - def threadexec(command, prefix = nil, color = nil) if prefix.nil? @@ -33,7 +17,19 @@ def threadexec(command, prefix = nil, color = nil) end Thread.new { - exec(command, prefix, color) + Open3.popen3(command) do |_, stdout, stderr, _| + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_out = stdout.gets + say_status prefix, line_out, color + end + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_err = stderr.gets + say_status prefix, line_err, :red + end + + end } end @@ -49,9 +45,21 @@ def forkexec(command, prefix = nil, color = nil) color = :cyan end - Process.fork do - exec(command, prefix, color) - end + Process.fork { + Open3.popen3(command) do |_, stdout, stderr, _| + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_out = stdout.gets + say_status prefix, line_out, color + end + + # noinspection RubyAssignmentExpressionInConditionalInspection + while line_err = stderr.gets + say_status prefix, line_err, :red + end + + end + } end From 309801be3801f6bd14028341dab89ca1c1ddd0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 25 Jul 2016 17:16:55 +0200 Subject: [PATCH 15/28] [TASK] Add some inline documentation -https://github.com/EugenMayer/docker-sync/pull/64#discussion_r71987529 -https://github.com/EugenMayer/docker-sync/pull/64#discussion_r71987556 -https://github.com/EugenMayer/docker-sync/pull/64#issuecomment-234721905 --- lib/docker-sync/execution.rb | 2 ++ lib/docker-sync/sync_manager.rb | 6 ++++++ lib/docker-sync/watch_strategy/unison.rb | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/lib/docker-sync/execution.rb b/lib/docker-sync/execution.rb index 15588788..ae62ef8b 100644 --- a/lib/docker-sync/execution.rb +++ b/lib/docker-sync/execution.rb @@ -34,6 +34,8 @@ def threadexec(command, prefix = nil, color = nil) end + # unison doesn't work when ran in a new thread + # this functions creates a full new process instead def forkexec(command, prefix = nil, color = nil) if prefix.nil? diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index 738b8a33..4b34e974 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -132,6 +132,12 @@ def join_threads end end + # when docker-sync runs with unison:unox processes created + # are not new threads but differents processes + # the previous .join method has no effect to keep the process + # running. If the main process exits, sync will continue to work + # but we have no way to kill created sync processes. This sleep + # prevents this. sleep unless has_threads rescue SystemExit, Interrupt diff --git a/lib/docker-sync/watch_strategy/unison.rb b/lib/docker-sync/watch_strategy/unison.rb index 0d2bf10a..ed591df6 100644 --- a/lib/docker-sync/watch_strategy/unison.rb +++ b/lib/docker-sync/watch_strategy/unison.rb @@ -16,6 +16,10 @@ def initialize(sync_name, options) @options = options @sync_name = sync_name @watch_fork = nil + # instantiate the sync task to easily access all common parameters between + # unison sync and watch + # basically unison watch is the command with the additionnal -repeat watch option + # note: this doesn't run a sync @unison = Docker_Sync::SyncStrategy::Unison_Unox.new(@sync_name, @options) end From c7514c307d25454d435ccb887a0047cc5b81ee59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 25 Jul 2016 17:42:53 +0200 Subject: [PATCH 16/28] [FEATURE] Increase maximum inotify watchers - in container only --- example/docker-sync.yml | 6 ++++++ lib/docker-sync/sync_strategy/unison-unox.rb | 3 +++ 2 files changed, 9 insertions(+) diff --git a/example/docker-sync.yml b/example/docker-sync.yml index 60deddab..4ee29a99 100644 --- a/example/docker-sync.yml +++ b/example/docker-sync.yml @@ -87,5 +87,11 @@ syncs: sync_strategy: 'unison-unox' sync_user: 'www-data' sync_userid: '33' + # If you need to sync a lot of files, you can reach out the system limit + # of inotify watches. You can set the limit by using this parameter (affects + # currently only the container side. see sudo sysctl -w kern.maxfiles=XXX, + # sudo sysctl -w kern.maxfilesperproc=XXX and ulimit -n XXXX + # to increase this limit on OSX on the host side). + max_inotify_watches: 100000 diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 5c9f0c37..50c3af7d 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -93,6 +93,7 @@ def start_container env['UNISON_EXCLUDES'] = @options['sync_excludes'].map { |pattern| "-ignore='Path #{pattern}'" }.join(' ') if @options.key?('sync_excludes') env['UNISON_OWNER'] = @options['sync_user'] if @options.key?('sync_user') + env['MAX_INOTIFY_WATCHES'] = @options['max_inotify_watches'] if @options.key?('max_inotify_watches') if @options['sync_userid'] == 'from_host' env['UNISON_OWNER_UID'] = Process.uid else @@ -106,10 +107,12 @@ def start_container exists = `docker ps --filter "status=exited" --filter "name=#{container_name}" | grep #{container_name}` if exists == '' say_status 'ok', "creating #{container_name} container", :white if @options['verbose'] + run_privileged = '--privileged' if @options.key?('max_inotify_watches') #TODO: replace by the minimum capabilities required cmd = "docker run -p '127.0.0.1::#{UNISON_CONTAINER_PORT}' \ -v #{volume_name}:#{@options['dest']} \ -e UNISON_DIR=#{@options['dest']} \ #{additional_docker_env} \ + #{run_privileged} \ --name #{container_name} \ -d #{@docker_image}" else From 55a75a4254a94fc574022118cee0fb69777d9b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 25 Jul 2016 21:18:38 +0200 Subject: [PATCH 17/28] [FEATURE] Restart unison server on start --- lib/docker-sync/sync_strategy/unison-unox.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 50c3af7d..858412e2 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -117,13 +117,14 @@ def start_container -d #{@docker_image}" else say_status 'ok', "starting #{container_name} container", :white if @options['verbose'] - cmd = "docker start #{container_name}" + cmd = "docker start #{container_name} && docker exec #{container_name} supervisorctl restart unison" end - say_status 'command', cmd, :white if @options['verbose'] - `#{cmd}` || raise('Start failed') else - say_status 'ok', "#{container_name} container still running", :blue + say_status 'ok', "#{container_name} container still running, restarting unison in container", :blue + cmd = "docker exec #{container_name} supervisorctl restart unison" end + say_status 'command', cmd, :white if @options['verbose'] + `#{cmd}` || raise('Start failed') say_status 'ok', "starting initial sync of #{container_name}", :white if @options['verbose'] # this sleep is needed since the container could be not started sleep 5 # TODO: replace with unison -testserver From 282818bb99a67b6ff0cb0d52a5e4297902094569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 25 Jul 2016 21:23:59 +0200 Subject: [PATCH 18/28] [FEATURE] Add terminal notifier support --- lib/docker-sync/sync_strategy/unison-unox.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 858412e2..46f625e8 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -3,6 +3,7 @@ require 'docker-sync/execution' require 'open3' require 'socket' +require 'terminal-notifier' module Docker_Sync module SyncStrategy @@ -59,6 +60,7 @@ def sync say_status 'error', "Error starting sync, exit code #{$?.exitstatus}", :red say_status 'message', stderr else + TerminalNotifier.notify("Synced #{@options['src']}", :title => 'Docker-Sync') say_status 'ok', "Synced #{@options['src']}", :white if @options['verbose'] say_status 'output', stdout From 75f35d0af1a18670b603c940aa9c6e1830777a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 26 Jul 2016 14:23:53 +0200 Subject: [PATCH 19/28] [REFACTORING] Replace ugly sleep with process.wait --- lib/docker-sync/sync_manager.rb | 13 +++---------- lib/docker-sync/sync_process.rb | 4 ++++ lib/docker-sync/watch_strategy/unison.rb | 4 ++++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index 4b34e974..6d6176ec 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -124,22 +124,15 @@ def run(sync_name = nil) def join_threads begin - has_threads = false @sync_processes.each do |sync_process| if sync_process.watch_thread sync_process.watch_thread.join - has_threads = true + end + if sync_process.watch_fork + Process.wait(sync_process.watch_fork) end end - # when docker-sync runs with unison:unox processes created - # are not new threads but differents processes - # the previous .join method has no effect to keep the process - # running. If the main process exits, sync will continue to work - # but we have no way to kill created sync processes. This sleep - # prevents this. - sleep unless has_threads - rescue SystemExit, Interrupt say_status 'shutdown', 'Shutting down...', :blue @sync_processes.each do |sync_process| diff --git a/lib/docker-sync/sync_process.rb b/lib/docker-sync/sync_process.rb index e4a54e0c..05b899d9 100644 --- a/lib/docker-sync/sync_process.rb +++ b/lib/docker-sync/sync_process.rb @@ -100,6 +100,10 @@ def watch @watch_strategy.run end + def watch_fork + return @watch_strategy.watch_fork + end + def watch_thread return @watch_strategy.watch_thread end diff --git a/lib/docker-sync/watch_strategy/unison.rb b/lib/docker-sync/watch_strategy/unison.rb index ed591df6..d2f52cec 100644 --- a/lib/docker-sync/watch_strategy/unison.rb +++ b/lib/docker-sync/watch_strategy/unison.rb @@ -41,6 +41,10 @@ def watch def watch_options end + def watch_fork + return @watch_fork + end + def watch_thread return nil end From c254f6374373d18ff69b7b955d33388b6e2e20a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 26 Jul 2016 14:32:12 +0200 Subject: [PATCH 20/28] [FEATURE] Manage watcher limit on the host side --- lib/docker-sync/sync_strategy/unison-unox.rb | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 46f625e8..058f76f4 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -36,14 +36,36 @@ def initialize(sync_name, options) end def run + increase_watcher_limit if @options.key?('max_inotify_watches') start_container sync end + def increase_watcher_limit + current_max_files_per_proc = `sysctl kern.maxfilesperproc | awk '{print $2}'` + if current_max_files_per_proc.to_f < @options['max_inotify_watches'] + cmd = 'sudo sysctl -w kern.maxfilesperproc=' + @options['max_inotify_watches'].to_s + say_status 'command', cmd, :white + `#{cmd}` || raise('Unable to increase maxfilesperproc') + else + say_status 'command', 'Current maxfilesperproc set to ' + current_max_files_per_proc.to_s, :white + end + current_max_files = `sysctl kern.maxfiles | awk '{print $2}'` + if current_max_files.to_f < @options['max_inotify_watches'] + cmd = 'sudo sysctl -w kern.maxfiles=' + @options['max_inotify_watches'].to_s + say_status 'command', cmd, :white + `#{cmd}` || raise('Unable to increase maxfiles') + else + say_status 'command', 'Current maxfiles set to ' + current_max_files.to_s, :white + end + end + def watch args = sync_options args.push("-repeat watch") - cmd = 'unison ' + args.join(' ') + cmd = '' + cmd = cmd + 'ulimit -n ' + @options['max_inotify_watches'].to_s + ' && ' if @options.key?('max_inotify_watches') + cmd = cmd + 'unison ' + args.join(' ') say_status 'command', cmd, :white if @options['verbose'] forkexec(cmd, "Sync #{@sync_name}", :blue) From e3ecf794bec0bceb3951d699b6da0fd04c098fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 26 Jul 2016 16:06:30 +0200 Subject: [PATCH 21/28] [BUGFIX] Unison doesn't catch file events - PB: when run with popen3 unison stopped to cath file events suddently. - FIX: using backticks did the trick. - TODO: understand why this happens --- lib/docker-sync/execution.rb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/docker-sync/execution.rb b/lib/docker-sync/execution.rb index ae62ef8b..9232f7d4 100644 --- a/lib/docker-sync/execution.rb +++ b/lib/docker-sync/execution.rb @@ -48,19 +48,7 @@ def forkexec(command, prefix = nil, color = nil) end Process.fork { - Open3.popen3(command) do |_, stdout, stderr, _| - - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_out = stdout.gets - say_status prefix, line_out, color - end - - # noinspection RubyAssignmentExpressionInConditionalInspection - while line_err = stderr.gets - say_status prefix, line_err, :red - end - - end + `#{command}` || raise(command + ' failed') } end From e94d9e623768eae9ff358bfb3190326e50b7e2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 1 Aug 2016 21:50:02 +0200 Subject: [PATCH 22/28] [TASK] Set default image as unison --- lib/docker-sync/sync_strategy/unison-unox.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison-unox.rb index 058f76f4..e41f1da6 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison-unox.rb @@ -23,7 +23,7 @@ def initialize(sync_name, options) if @options.key?('image') @docker_image = @options['image'] else - @docker_image = 'eugenmayer/unison:unox' + @docker_image = 'eugenmayer/unison' end begin Preconditions::unison_available From 27c59286c5b6e09172505aec7d755fbb1b4587eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 1 Aug 2016 21:54:30 +0200 Subject: [PATCH 23/28] [TASK] Info how to install unox in error message --- lib/docker-sync/preconditions.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/docker-sync/preconditions.rb b/lib/docker-sync/preconditions.rb index b6e7e574..1688c480 100644 --- a/lib/docker-sync/preconditions.rb +++ b/lib/docker-sync/preconditions.rb @@ -46,7 +46,9 @@ def self.unison_available def self.unox_available if (find_executable0 'unison-fsmonitor').nil? - raise('Could not find unison-fsmonitor binary in path. Please install it, see https://github.com/hnsl/unox') + raise('Could not find unison-fsmonitor binary in path. Please install it, see https://github.com/hnsl/unox, or simply run : + wget -O /usr/local/bin/unison-fsmonitor https://raw.githubusercontent.com/hnsl/unox/master/unox.py \ + && chmod +x /usr/local/bin/unison-fsmonitor') end end end \ No newline at end of file From 224ecaffd0f184d627de4e4e0e39c921f42417d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 1 Aug 2016 23:20:42 +0200 Subject: [PATCH 24/28] [TASK] Rename unison-unox to unison --- lib/docker-sync/sync_manager.rb | 2 +- lib/docker-sync/sync_process.rb | 10 +++++----- .../sync_strategy/{unison-unox.rb => unison.rb} | 2 +- lib/docker-sync/watch_strategy/unison.rb | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) rename lib/docker-sync/sync_strategy/{unison-unox.rb => unison.rb} (99%) diff --git a/lib/docker-sync/sync_manager.rb b/lib/docker-sync/sync_manager.rb index 0f4b9107..9d346666 100644 --- a/lib/docker-sync/sync_manager.rb +++ b/lib/docker-sync/sync_manager.rb @@ -80,7 +80,7 @@ def validate_config(config) def validate_sync_config(name, sync_config) config_mandatory = %w[src dest] - config_mandatory.push('sync_host_port') unless sync_config['sync_strategy'] == 'unison-unox' #TODO: Implement autodisovery for other strategies + config_mandatory.push('sync_host_port') unless sync_config['sync_strategy'] == 'unison' #TODO: Implement autodisovery for other strategies config_mandatory.each do |key| raise ("#{name} does not have #{key} condiguration value set - this is mandatory") unless sync_config.key?(key) end diff --git a/lib/docker-sync/sync_process.rb b/lib/docker-sync/sync_process.rb index 5bd14625..184f1676 100644 --- a/lib/docker-sync/sync_process.rb +++ b/lib/docker-sync/sync_process.rb @@ -1,8 +1,8 @@ require 'thor/shell' # noinspection RubyResolve require 'docker-sync/sync_strategy/rsync' +require 'docker-sync/sync_strategy/unison-onesided' require 'docker-sync/sync_strategy/unison' -require 'docker-sync/sync_strategy/unison-unox' # noinspection RubyResolve require 'docker-sync/watch_strategy/fswatch' require 'docker-sync/watch_strategy/dummy' @@ -38,8 +38,8 @@ def set_sync_strategy @sync_strategy = Docker_Sync::SyncStrategy::Rsync.new(@sync_name, @options) when 'unison-onesided' @sync_strategy = Docker_Sync::SyncStrategy::Unison_Onesided.new(@sync_name, @options) - when 'unison-unox' - @sync_strategy = Docker_Sync::SyncStrategy::Unison_Unox.new(@sync_name, @options) + when 'unison' + @sync_strategy = Docker_Sync::SyncStrategy::Unison.new(@sync_name, @options) else @sync_strategy = Docker_Sync::SyncStrategy::Rsync.new(@sync_name, @options) end @@ -61,10 +61,10 @@ def set_watch_strategy @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) end else - if @options['sync_strategy'] == 'unison-unox' + if @options['sync_strategy'] == 'unison' @watch_strategy = Docker_Sync::WatchStrategy::Unison.new(@sync_name, @options) else - @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) + @watch_strategy = Docker_Sync::WatchStrategy::Fswatch.new(@sync_name, @options) end end end diff --git a/lib/docker-sync/sync_strategy/unison-unox.rb b/lib/docker-sync/sync_strategy/unison.rb similarity index 99% rename from lib/docker-sync/sync_strategy/unison-unox.rb rename to lib/docker-sync/sync_strategy/unison.rb index e41f1da6..eafea9e9 100644 --- a/lib/docker-sync/sync_strategy/unison-unox.rb +++ b/lib/docker-sync/sync_strategy/unison.rb @@ -7,7 +7,7 @@ module Docker_Sync module SyncStrategy - class Unison_Unox + class Unison include Thor::Shell include Execution diff --git a/lib/docker-sync/watch_strategy/unison.rb b/lib/docker-sync/watch_strategy/unison.rb index d2f52cec..1fa681b5 100644 --- a/lib/docker-sync/watch_strategy/unison.rb +++ b/lib/docker-sync/watch_strategy/unison.rb @@ -1,7 +1,7 @@ require 'thor/shell' require 'docker-sync/execution' require 'docker-sync/preconditions' -require 'docker-sync/sync_strategy/unison-unox' +require 'docker-sync/sync_strategy/unison' module Docker_Sync module WatchStrategy @@ -20,7 +20,7 @@ def initialize(sync_name, options) # unison sync and watch # basically unison watch is the command with the additionnal -repeat watch option # note: this doesn't run a sync - @unison = Docker_Sync::SyncStrategy::Unison_Unox.new(@sync_name, @options) + @unison = Docker_Sync::SyncStrategy::Unison.new(@sync_name, @options) end def run From 1884f1a8fc4d8010093c06b5f32f07666780d5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Mon, 1 Aug 2016 23:24:03 +0200 Subject: [PATCH 25/28] [TASK] Update and fix example --- example/docker-compose-dev.yml | 6 +++++- example/docker-compose.yml | 12 ++++++++---- example/docker-sync.yml | 12 ++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/example/docker-compose-dev.yml b/example/docker-compose-dev.yml index 6748cfd9..f691a09c 100644 --- a/example/docker-compose-dev.yml +++ b/example/docker-compose-dev.yml @@ -7,11 +7,15 @@ services: volumes_from: - container:rsync-sync:rw # will be mounted on /var/www unisononsidedapp: + volumes_from: + - container:unison-onesided-sync:rw # will be mounted on /app/code + unison: volumes_from: - container:unison-sync:rw # will be mounted on /app/code - volumes: rsync-sync: external: true unison-onesided-sync: external: true + unison-sync: + external: true diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 52432b88..f0025e43 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -2,11 +2,15 @@ version: "2" services: - someapp: + rsyncapp: image: alpine - container_name: 'fullexample_app' + container_name: 'rsyncapp' command: ['watch', '-n1', 'cat /var/www/somefile.txt'] - otherapp: + unisononsidedapp: image: alpine - container_name: 'simpleexample_app' + container_name: 'unisononsidedapp' + command: ['watch', '-n1', 'cat /app/code/somefile.txt'] + unison: + image: alpine + container_name: 'unison' command: ['watch', '-n1', 'cat /app/code/somefile.txt'] diff --git a/example/docker-sync.yml b/example/docker-sync.yml index 82f164fa..44198e95 100644 --- a/example/docker-sync.yml +++ b/example/docker-sync.yml @@ -64,3 +64,15 @@ syncs: sync_host_ip: 'localhost' sync_host_port: 20872 sync_strategy: 'unison-onesided' # this time we pick unison-onesided + unison-sync: + src: './data4/' + dest: '/data' + image: 'mickaelperrin/docker-unison-unox' + sync_host_ip: 'localhost' + sync_strategy: 'unison' + sync_user: 'www-data' + sync_userid: '33' + # If you need to sync a lot of files, you can reach out the system limit + # of inotify watches. You can set the limit by using this parameter. This will + # prompt you for your sudo password to modify the system configuration. + #max_inotify_watches: 100000 From b569ac27c3ecc42339e420649cbd3e902df7f06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 2 Aug 2016 10:42:48 +0200 Subject: [PATCH 26/28] [TASK] Revert default image to unison:unox --- lib/docker-sync/sync_strategy/unison.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docker-sync/sync_strategy/unison.rb b/lib/docker-sync/sync_strategy/unison.rb index eafea9e9..e02578c4 100644 --- a/lib/docker-sync/sync_strategy/unison.rb +++ b/lib/docker-sync/sync_strategy/unison.rb @@ -23,7 +23,7 @@ def initialize(sync_name, options) if @options.key?('image') @docker_image = @options['image'] else - @docker_image = 'eugenmayer/unison' + @docker_image = 'eugenmayer/unison:unox' end begin Preconditions::unison_available From 84baa85f5178b513e2667222d11f7c9f1b4f5c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 2 Aug 2016 10:46:29 +0200 Subject: [PATCH 27/28] [TASK] Use curl instead of wget in unox install curl is default on Mac OS X, wget needs MacPorts --- lib/docker-sync/preconditions.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docker-sync/preconditions.rb b/lib/docker-sync/preconditions.rb index 1688c480..694464a8 100644 --- a/lib/docker-sync/preconditions.rb +++ b/lib/docker-sync/preconditions.rb @@ -47,7 +47,7 @@ def self.unison_available def self.unox_available if (find_executable0 'unison-fsmonitor').nil? raise('Could not find unison-fsmonitor binary in path. Please install it, see https://github.com/hnsl/unox, or simply run : - wget -O /usr/local/bin/unison-fsmonitor https://raw.githubusercontent.com/hnsl/unox/master/unox.py \ + curl "https://raw.githubusercontent.com/hnsl/unox/master/unox.py" -o "/usr/local/bin/unison-fsmonitor" \ && chmod +x /usr/local/bin/unison-fsmonitor') end end From 5f288f2e205caae5b8bcba58fa2b6b9a1d7c5a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Perrin?= Date: Tue, 2 Aug 2016 12:06:05 +0200 Subject: [PATCH 28/28] [FEATURE] Auto install of unox --- lib/docker-sync/preconditions.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/docker-sync/preconditions.rb b/lib/docker-sync/preconditions.rb index 694464a8..bc0a404d 100644 --- a/lib/docker-sync/preconditions.rb +++ b/lib/docker-sync/preconditions.rb @@ -46,9 +46,14 @@ def self.unison_available def self.unox_available if (find_executable0 'unison-fsmonitor').nil? - raise('Could not find unison-fsmonitor binary in path. Please install it, see https://github.com/hnsl/unox, or simply run : - curl "https://raw.githubusercontent.com/hnsl/unox/master/unox.py" -o "/usr/local/bin/unison-fsmonitor" \ - && chmod +x /usr/local/bin/unison-fsmonitor') + cmd = 'curl "https://raw.githubusercontent.com/hnsl/unox/master/unox.py" -o "/usr/local/bin/unison-fsmonitor" \ + && chmod +x /usr/local/bin/unison-fsmonitor' + Thor::Shell::Basic.new.say_status 'warning', "Could not find unison-fsmonitor binary in path. Please install unox before you continue, see https://github.com/hnsl/unox.", :yellow + if Thor::Shell::Basic.new.yes?("Shall I install unison-fsmonitor for you?") + `#{cmd}` + else + raise("Please install it, see https://github.com/hnsl/unox, or simply run :\n #{cmd}") + end end end -end \ No newline at end of file +end