[to #421] add a python script to analyze slow log stats (#527) (#534)

Co-authored-by: Jian Zhang <[email protected]>

.github/license-checker.yml

@@ -0,0 +1,26 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: TiKV Project Authors.
+  paths-ignore:
+    - 'config/'
+    - 'dev/'
+    - 'docs/'
+    - 'metrics/'
+    - 'LICENSE'
+    - 'Makefile'
+    - 'pom.xml'
+    - 'shell.nix'
+    - '.ci/'
+    - '.gitignore'
+    - '.gitattributes'
+    - '.github/'
+    - '**/*.md'
+    - '**/*.properties'
+    - '**/*.json'
+    - '**/*.pem'
+    - '**/*.crt'
+    - '**/*.g4'
+    - 'src/main/java/io/grpc/'
+    - 'src/main/java/io/netty/'
+  comment: on-failure

dev/README.md

@@ -1,8 +1,6 @@
-# TiSpark Dev Tools Guide
+# TiKV Java Client Dev Tools
 
-## Formatting
-
-### Java Format
+## Code Formatting
 
 TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build.
 
@@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https:
 
     ```shell script
    ./dev/javafmt
-    ```
+    ```

pom.xml

@@ -278,7 +278,7 @@
                     <execution>
                         <id>clone proto files</id>
                         <configuration>
-                            <executable>${basedir}/scripts/proto.sh</executable>
+                            <executable>${basedir}/dev/proto.sh</executable>
                         </configuration>
                         <phase>validate</phase>
                         <goals>

scripts/rstats.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+#!coding:utf-8
+
+# Copyright 2022 TiKV Project Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+import json
+import argparse
+
+def main():
+    pattern = r'.*SlowLog:.*'
+    slowstr = "SlowLog:"
+    grpc_pattern = "gRPC tikvpb.Tikv"
+    backoff_pattern = "backoff "
+
+    args = parse_args()
+    items = []
+    with open(args.slowlog, encoding = 'utf-8') as f:
+        for line in f.readlines():
+            matched = re.match(pattern, line, re.M|re.I)
+            if matched is not None:
+                log = json.loads(line[(line.index(slowstr) + len(slowstr)):])
+                item = {
+                    'req': log['func'],
+                    'start': log['start'],
+                    'tot_lat': latency_ms(log),
+                    'tot_grpc': 0,
+                    'tot_bo': 0,
+                }
+                items.append(item)
+                for span in log['spans']:
+                    if grpc_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_grpc'] += latency_ms(span)
+                    elif backoff_pattern in span['name'] and span['duration'] != 'N/A':
+                        item['tot_bo'] += latency_ms(span)
+
+        if args.order == "total":
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+        elif args.order == "grpc":
+            items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True)
+        elif args.order == "backoff":
+            items = sorted(items, key=lambda d: d['tot_bo'], reverse=True)
+        else:
+            print("unsupported order option, use default value: total")
+            items = sorted(items, key=lambda d: d['tot_lat'], reverse=True)
+
+    fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}"
+    print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)"))
+    for item in items:
+        print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo']))
+
+def latency_ms(span):
+    return int(span['duration'][:len(span['duration'])-2])
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer")
+    parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff")
+    parser.add_argument("slowlog", help="slow log file")
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    main()
+