From 0bbe5f0cd5c91c6499d1ac531732948a9c00a1f7 Mon Sep 17 00:00:00 2001 From: okJiang Date: Mon, 10 Apr 2023 18:05:00 +0800 Subject: [PATCH] sync_diff: fix hidden column (#712) close pingcap/tidb-tools#713 --- go.mod1 | 24 ++++----- go.sum1 | 48 ++++++++--------- sync_diff_inspector/splitter/bucket.go | 8 +++ sync_diff_inspector/splitter/random.go | 22 ++++++-- sync_diff_inspector/utils/utils.go | 4 ++ .../expression/config.toml | 54 +++++++++++++++++++ tests/sync_diff_inspector/expression/run.sh | 23 ++++++++ 7 files changed, 142 insertions(+), 41 deletions(-) create mode 100644 tests/sync_diff_inspector/expression/config.toml create mode 100644 tests/sync_diff_inspector/expression/run.sh diff --git a/go.mod1 b/go.mod1 index 491bcb14c..c1c7b9056 100644 --- a/go.mod1 +++ b/go.mod1 @@ -10,11 +10,11 @@ require ( github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c - github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 + github.com/pingcap/kvproto v0.0.0-20230312142449-01623096c924 github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c - github.com/pingcap/tidb v1.1.0-beta.0.20230310043312-53f15f6ed774 - github.com/pingcap/tidb/parser v0.0.0-20230310043312-53f15f6ed774 - github.com/pingcap/tiflow v0.0.0-20230322131442-4e0d18c1cb34 + github.com/pingcap/tidb v1.1.0-beta.0.20230317053715-5aceb2e525f6 + github.com/pingcap/tidb/parser v0.0.0-20230317053715-5aceb2e525f6 + github.com/pingcap/tiflow v0.0.0-20230403075857-97664e66e63c github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.2 @@ -160,7 +160,7 @@ require ( github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 // indirect github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 // indirect - github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b // indirect + github.com/pingcap/tipb v0.0.0-20230310043643-5362260ee6f7 // indirect github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -187,7 +187,7 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/client-go/v2 v2.0.7-0.20230309100832-f555fdd2c9d8 // indirect + github.com/tikv/client-go/v2 v2.0.7-0.20230313133219-c9119d02cef7 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect @@ -221,16 +221,16 @@ require ( go.opentelemetry.io/otel/trace v0.20.0 // indirect go.opentelemetry.io/proto/otlp v0.7.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.6.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.106.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum1 b/go.sum1 index b03810595..ccab40a94 100644 --- a/go.sum1 +++ b/go.sum1 @@ -715,8 +715,8 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= +github.com/pingcap/kvproto v0.0.0-20230312142449-01623096c924 h1:z6WwBPP0Txmal5zf+H7vf/lSmKZtSS8BTNwiLjEjdnA= +github.com/pingcap/kvproto v0.0.0-20230312142449-01623096c924/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= @@ -724,15 +724,15 @@ github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYh github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= -github.com/pingcap/tidb v1.1.0-beta.0.20230310043312-53f15f6ed774 h1:Ul8TORvRqbXRH7DsM58bIQQc5DJMG7o1M+lbmp3rkKY= -github.com/pingcap/tidb v1.1.0-beta.0.20230310043312-53f15f6ed774/go.mod h1:BEY/Sj8O09HTaAU97C/7Afp+mHEu/iwAxyM6N7eXnjk= +github.com/pingcap/tidb v1.1.0-beta.0.20230317053715-5aceb2e525f6 h1:trQrA+zPQQg9EFI8oU/FbpWkafH/wFJwupYx/PXr2N4= +github.com/pingcap/tidb v1.1.0-beta.0.20230317053715-5aceb2e525f6/go.mod h1:cRBtPNq5rhGQWTX+aVWMgMNGfMHU6QD/5glJr38g5uk= github.com/pingcap/tidb/parser v0.0.0-20221126021158-6b02a5d8ba7d/go.mod h1:ElJiub4lRy6UZDb+0JHDkGEdr6aOli+ykhyej7VCLoI= -github.com/pingcap/tidb/parser v0.0.0-20230310043312-53f15f6ed774 h1:yn5uxguBmI9wcfc3HLfoCVj633OPQXMabnoFVAf2rcM= -github.com/pingcap/tidb/parser v0.0.0-20230310043312-53f15f6ed774/go.mod h1:IxXRBZ14Of1KkR3NXEwsoKrM8JbkOIHJHpwS/Ad8vPY= -github.com/pingcap/tiflow v0.0.0-20230322131442-4e0d18c1cb34 h1:mbGi2bqO+fRKe37G0YW8sJk/oll8K7NzUqWtCukwrbU= -github.com/pingcap/tiflow v0.0.0-20230322131442-4e0d18c1cb34/go.mod h1:bqT/K4Y/0f9u2EocWcmmDnq6Dko3+cCAsxIz/iyE6EU= -github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b h1:j5sw2YZY7QfgIFZEoUcn1P5cYflms1PCVVS96i+IQiI= -github.com/pingcap/tipb v0.0.0-20230119054146-c6b7a5a1623b/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tidb/parser v0.0.0-20230317053715-5aceb2e525f6 h1:puP9sVOVIUmOzJZtN/Vv0H1ih8VAax7BBNUqLZEZFrs= +github.com/pingcap/tidb/parser v0.0.0-20230317053715-5aceb2e525f6/go.mod h1:IxXRBZ14Of1KkR3NXEwsoKrM8JbkOIHJHpwS/Ad8vPY= +github.com/pingcap/tiflow v0.0.0-20230403075857-97664e66e63c h1:E06eByN/jK0pGFmUMGHRADLTcearOrODQuGQMs+Sevg= +github.com/pingcap/tiflow v0.0.0-20230403075857-97664e66e63c/go.mod h1:bqT/K4Y/0f9u2EocWcmmDnq6Dko3+cCAsxIz/iyE6EU= +github.com/pingcap/tipb v0.0.0-20230310043643-5362260ee6f7 h1:CeeMOq1aHPAhXrw4eYXtQRyWOFlbfqK1+3f9Iop4IfU= +github.com/pingcap/tipb v0.0.0-20230310043643-5362260ee6f7/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -875,8 +875,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tikv/client-go/v2 v2.0.7-0.20230309100832-f555fdd2c9d8 h1:TKxKhSpF+G6yAdfbPo2Nl2vl6wP8Tm1gufU1HPnL6u0= -github.com/tikv/client-go/v2 v2.0.7-0.20230309100832-f555fdd2c9d8/go.mod h1:ti7e5tuvVBJz5UKlPBR2OHEBC7sOzKsetfN8STZ+KpQ= +github.com/tikv/client-go/v2 v2.0.7-0.20230313133219-c9119d02cef7 h1:1Xqx7UgNloQqQvR8k3oHGrPDlgYVK4XQnwZacfPbAQA= +github.com/tikv/client-go/v2 v2.0.7-0.20230313133219-c9119d02cef7/go.mod h1:61YdH33t2SXn7kOGgwHHbyif+O9PfFIVdLezu7gDQ6c= github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 h1:vCVu7LxFou5WuaY6jHDMHKVeJTtwr5o2i1xWgGAdDo4= github.com/tikv/pd/client v0.0.0-20230309025512-47cd76ae5d67 h1:AXgc/Ij348pp0TsMPq/tmQA4O0EOAGntTKzB1imhpcU= github.com/tikv/pd/client v0.0.0-20230309025512-47cd76ae5d67/go.mod h1:N2QHc05Vll8CofXQor47lpW5d22WDosFC8WPVx9BsbU= @@ -1047,8 +1047,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1136,16 +1136,16 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1237,13 +1237,13 @@ golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1318,8 +1318,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/sync_diff_inspector/splitter/bucket.go b/sync_diff_inspector/splitter/bucket.go index eff61fa0e..73357af26 100644 --- a/sync_diff_inspector/splitter/bucket.go +++ b/sync_diff_inspector/splitter/bucket.go @@ -152,6 +152,7 @@ func (s *BucketIterator) init(ctx context.Context, startRange *RangeInfo) error indices = dbutil.FindAllIndex(s.table.Info) } +NEXTINDEX: for _, index := range indices { if index == nil { continue @@ -172,6 +173,13 @@ func (s *BucketIterator) init(ctx context.Context, startRange *RangeInfo) error continue } + // skip the index that has expression column + for _, col := range indexColumns { + if col.Hidden { + continue NEXTINDEX + } + } + bucket, ok := buckets[index.Name.O] if !ok { // We found an index matching the "index-fields", but no bucket is found diff --git a/sync_diff_inspector/splitter/random.go b/sync_diff_inspector/splitter/random.go index 2976c2c49..43561cac6 100644 --- a/sync_diff_inspector/splitter/random.go +++ b/sync_diff_inspector/splitter/random.go @@ -166,7 +166,6 @@ func (s *RandomIterator) Close() { // GetSplitFields returns fields to split chunks, order by pk, uk, index, columns. func GetSplitFields(table *model.TableInfo, splitFields []string) ([]*model.ColumnInfo, error) { - cols := make([]*model.ColumnInfo, 0, len(table.Columns)) colsMap := make(map[string]*model.ColumnInfo) splitCols := make([]*model.ColumnInfo, 0, 2) @@ -188,13 +187,26 @@ func GetSplitFields(table *model.TableInfo, splitFields []string) ([]*model.Colu } indices := dbutil.FindAllIndex(table) if len(indices) != 0 { - for _, col := range indices[0].Columns { - cols = append(cols, colsMap[col.Name.O]) + NEXTINDEX: + for _, idx := range indices { + cols := make([]*model.ColumnInfo, 0, len(table.Columns)) + for _, icol := range idx.Columns { + col := colsMap[icol.Name.O] + if col.Hidden { + continue NEXTINDEX + } + cols = append(cols, col) + } + return cols, nil } - return cols, nil } - return []*model.ColumnInfo{table.Columns[0]}, nil + for _, col := range table.Columns { + if !col.Hidden { + return []*model.ColumnInfo{col}, nil + } + } + return nil, errors.NotFoundf("not found column") } // splitRangeByRandom splits a chunk to multiple chunks by random diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index fe96f2969..641e2827d 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -143,6 +143,10 @@ func GetTableRowsQueryFormat(schema, table string, tableInfo *model.TableInfo, c columnNames := make([]string, 0, len(tableInfo.Columns)) for _, col := range tableInfo.Columns { + if col.Hidden { + continue + } + name := dbutil.ColumnName(col.Name.O) // When col value is 0, the result is NULL. // But we can use ISNULL to distinguish between null and 0. diff --git a/tests/sync_diff_inspector/expression/config.toml b/tests/sync_diff_inspector/expression/config.toml new file mode 100644 index 000000000..782ffa978 --- /dev/null +++ b/tests/sync_diff_inspector/expression/config.toml @@ -0,0 +1,54 @@ +# Diff Configuration. + +######################### Global config ######################### + +# how many goroutines are created to check data +check-thread-count = 4 + +# set false if just want compare data by checksum, will skip select data when checksum is not equal. +# set true if want compare all different rows, will slow down the total compare time. +export-fix-sql = true + +# ignore check table's data +check-struct-only = false + +######################### Databases config ######################### +[data-sources] +[data-sources.tidb1] + host = "127.0.0.1" + port = 4001 + user = "root" + password = "" + # remove comment if use tidb's snapshot data + # snapshot = "2016-10-08 16:45:26" + +[data-sources.tidb] + host = "127.0.0.1" + port = 4000 + user = "root" + password = "" + # remove comment if use tidb's snapshot data + # snapshot = "2016-10-08 16:45:26" + +######################### Task config ######################### +[task] + # 1 fix sql: fix-target-TIDB1.sql + # 2 log: sync-diff.log + # 3 summary: summary.txt + # 4 checkpoint: a dir + output-dir = "/tmp/tidb_tools_test/sync_diff_inspector/output" + + source-instances = ["tidb1"] + + target-instance = "tidb" + + # tables need to check. + target-check-tables = ["expression_test.diff"] + + target-configs = ["config1"] + +######################### Table config ######################### +[table-configs.config1] +target-tables = ["test1.v"] +range = "TRUE" +chunk-size = 1 diff --git a/tests/sync_diff_inspector/expression/run.sh b/tests/sync_diff_inspector/expression/run.sh new file mode 100644 index 000000000..612fc24cb --- /dev/null +++ b/tests/sync_diff_inspector/expression/run.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -ex + +cd "$(dirname "$0")" +OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector/output +FIX_DIR=/tmp/tidb_tools_test/sync_diff_inspector/fixsql +rm -rf $OUT_DIR +rm -rf $FIX_DIR +mkdir -p $OUT_DIR +mkdir -p $FIX_DIR + +for port in 4000 4001; do + mysql -uroot -h 127.0.0.1 -P $port -e "create database if not exists expression_test;" + mysql -uroot -h 127.0.0.1 -P $port -e "create table expression_test.diff(\`a\`\`;sad\` int, id int);" + mysql -uroot -h 127.0.0.1 -P $port -e "alter table expression_test.diff add index i1((\`a\`\`;sad\` + 1 + \`a\`\`;sad\`));" + mysql -uroot -h 127.0.0.1 -P $port -e "insert into expression_test.diff values (1,1),(2,2),(3,3);" +done + +echo "check result should be pass" +sync_diff_inspector --config=./config.toml > $OUT_DIR/expression_diff.output +check_contains "check pass!!!" $OUT_DIR/sync_diff.log +rm -rf $OUT_DIR/*