restore-util: Implement split/scatter (#274)

* implement split/scatter

Signed-off-by: 5kbpers <[email protected]>

* init test

Signed-off-by: 5kbpers <[email protected]>

* redesign output/input of the lib

Signed-off-by: 5kbpers <[email protected]>

* update dependency

Signed-off-by: 5kbpers <[email protected]>

* add commments and more tests

Signed-off-by: 5kbpers <[email protected]>

* add ScanRegions interface to Client

Signed-off-by: 5kbpers <[email protected]>

* fix potential data race

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* Apply suggestions from code review

Co-Authored-By: kennytm <[email protected]>

* Update pkg/restore-util/client.go

Co-Authored-By: kennytm <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* update dependency

Signed-off-by: 5kbpers <[email protected]>

* resolve conflicts

Signed-off-by: 5kbpers <[email protected]>

* fix prefix rewrite

Signed-off-by: 5kbpers <[email protected]>

* add RewriteRule/skip failed scatter region/retry the SplitRegion

Signed-off-by: 5kbpers <[email protected]>

* fix test

Signed-off-by: 5kbpers <[email protected]>

* check if region has peer

Signed-off-by: 5kbpers <[email protected]>

* more logs

Signed-off-by: 5kbpers <[email protected]>

.gitignore

@@ -4,3 +4,4 @@
 deps.sh
 go.sum
 go.mod
+.idea/

go.mod1

@@ -4,22 +4,70 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DATA-DOG/go-sqlmock v1.3.3
 	github.com/Shopify/sarama v1.21.0
-	github.com/coreos/bbolt v1.3.1-coreos.6 // indirect
-	github.com/coreos/etcd v3.3.10+incompatible
-	github.com/go-sql-driver/mysql v1.4.0
-	github.com/golang/protobuf v1.2.0
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/coreos/etcd v3.3.16+incompatible
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/gogo/protobuf v1.3.0 // indirect
+	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
+	github.com/golang/protobuf v1.3.2
+	github.com/google/btree v1.0.0
+	github.com/google/go-cmp v0.3.1 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
+	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.11.3 // indirect
+	github.com/klauspost/cpuid v1.2.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.5.0 // indirect
+	github.com/onsi/ginkgo v1.10.2 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
-	github.com/pingcap/errors v0.11.1
-	github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562
-	github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596
-	github.com/pingcap/parser v0.0.0-20190312024907-3f6280b08c8b
-	github.com/pingcap/pd v2.1.0-rc.4+incompatible
-	github.com/pingcap/tidb v0.0.0-20190320062740-9071c7b5b9ed
-	github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7
+	github.com/pingcap/errcode v0.3.0 // indirect
+	github.com/pingcap/errors v0.11.4
+	github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 // indirect
+	github.com/pingcap/kvproto v0.0.0-20191011042334-8ee4fd8fb4ca
+	github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
+	github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6
+	github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0
+	github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7
+	github.com/pingcap/tipb v0.0.0-20191008064422-018b2fadf414
+	github.com/prometheus/client_golang v1.1.0 // indirect
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
+	github.com/shirou/gopsutil v2.19.9+incompatible // indirect
 	github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726
 	github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8
-	go.uber.org/atomic v1.3.2
-	go.uber.org/zap v1.9.1
-	golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e
-	google.golang.org/grpc v1.17.0
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
+	github.com/uber-go/atomic v1.4.0 // indirect
+	github.com/uber/jaeger-client-go v2.19.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
+	github.com/unrolled/render v1.0.1 // indirect
+	go.etcd.io/etcd v3.3.16+incompatible // indirect
+	go.uber.org/atomic v1.4.0
+	go.uber.org/multierr v1.2.0 // indirect
+	go.uber.org/zap v1.10.0
+	golang.org/x/crypto v0.0.0-20191010185427-af544f31c8ac // indirect
+	golang.org/x/net v0.0.0-20191009170851-d66e71096ffb
+	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
+	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect
+	golang.org/x/tools v0.0.0-20191010201905-e5ffc44a6fee // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 // indirect
+	google.golang.org/grpc v1.24.0
+	gopkg.in/yaml.v2 v2.2.4 // indirect
+	sigs.k8s.io/yaml v1.1.0 // indirect
+	sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 // indirect
 )
+
+replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190930215403-16217165b5de
+
+go 1.13