optimize for http empty request or response, fix differ, delete unuse…

…d code
didi · Apr 17, 2019 · 6246e43 · 6246e43
1 parent 38cab6e
commit 6246e43
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 24 deletions.
diff --git a/composer.json b/composer.json
@@ -3,7 +3,7 @@
   "description": "A PHP Replay Client For Rdebug",
   "keywords": ["php", "rdebug", "koala", "replay", "Real Debugger"],
   "type": "library",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "license": "Apache-2.0",
   "config": {
     "secure-http": false,

diff --git a/example/php/README.md b/example/php/README.md
@@ -89,7 +89,7 @@ Example code just curl `http://php.net/get-involved.php`.
 - [Compile koala-libc.so](https://github.com/didi/rdebug#42-compile-koala-libc)
 - [Compile koala-recorder.so](https://github.com/didi/rdebug#43-compile-koala)
 
-If you don't want to compile and record, this example also provide a recorded session [1548160113499755925-1158745](./1548160113499755925-1158745), you could go to [replay session](#replay-session).
+If you don't want to compile and record, this example also provide a recorded session [1548160113499755925-1158745](./1548160113499755925-1158745), just go to [replay session](#replay-session).
 
 ## Start Service
 
@@ -102,6 +102,7 @@ $ cp /path/to/koala-libc.so /usr/local/var/koala
 $ cp /path/to/koala-recorder.so /usr/local/var/koala 
 
 # Start php-fpm with koala-libc.so & koala-recorder.so (Inject *.so to php-fpm)
+# Make sure set `clear_env = no` in php-fpm.conf and `KOALA_RECORD_TO_DIR` directory can writeable.
 # macOS
 $ DYLD_INSERT_LIBRARIES="/usr/local/var/koala/koala-libc.so:/usr/lib/libcurl.dylib" DYLD_FORCE_FLAT_NAMESPACE="y" LC_CTYPE="C" KOALA_SO=/usr/local/var/koala/koala-recorder.so KOALA_RECORD_TO_DIR=/usr/local/var/koala /usr/local/sbin/php-fpm
 # or, Linux
@@ -169,12 +170,25 @@ $ /path/to/rdebug/php/midi/bin/midi run -f 1548160113499755925-1158745 -ORT
 $ midi.phar run -f 1548160113499755925-1158745 -ORT
 ```
 
+If your system is macOS, must make sure your SIP is disable when macOS version > `10.12.6`, otherwise `DYLD_INSERT_LIBRARIES` will not work.
+
+If midi replay fail, you could replay with `-vvv` options and will get the `cmd` command:
+
+```$xslt
+$ /path/to/rdebug/php/midi/bin/midi run -f 1548160113499755925-1158745 -ORT -vvv
+...
+Finish start replayer, cmd=`LD_PRELOAD="/tmp/midi/res/replayer/koala-replayer.so" LC_CTYPE="C" KOALA_INBOUND_ADDR=":5514" KOALA_SUT_ADDR="127.0.0.1:5515" KOALA_OUTBOUND_ADDR="127.0.0.1:5516" KOALA_REPLAYING_MATCH_STRATEGY="sim" KOALA_REPLAYING_MATCH_THRESHOLD="0.75" KOALA_LOG_LEVEL="TRACE" KOALA_LOG_FILE="/tmp/midi/log/koala.log" KOALA_INBOUND_READ_TIMEOUT="86400s" php -S 127.0.0.1:5515 -d memory_limit="2G" -d error_log="/tmp/midi/log/php-error.log" -d auto_prepend_file="/usr/local/var/koala/.midi/_internal_auto_prepend.php"`, pid=439.
+```
+
+Copy `cmd` command and run it at your console, you will get some useful information.
+
 ### Replay Options
 
 ```
 -R generate replay report
 -T generate xdebug trace report
 -C generate code coverage report
+-vvv display internal logs to console
 ```
 
 Use `midi run -h` to see more options.
@@ -278,7 +292,7 @@ Config [Documentation](https://github.com/didi/rdebug/blob/master/doc/midi/Confi
 # Check fpm master, will get koala-libc.so
 $ lsof -p php-fpm-master-PID | grep koala
 
-# Check php-worker after request, will koala-libc.so & koala-recorder.so
+# Check php-worker after request, will get koala-libc.so & koala-recorder.so
 $ lsof -p php-fpm-worker-PID | grep koala
 ```
 
@@ -288,3 +302,5 @@ Or, open koala debug log:
 # Before start php-fpm with koala, export env:
 $ export KOALA_LOG_FILE="/tmp/rdebug/debug.log" KOALA_LOG_LEVEL="DEBUG"
 ```
+
+If everything is ok, but can not find recorded session files. Make sure your `KOALA_RECORD_TO_DIR` directory can writeable.
diff --git a/koala/outbound/mysql.go b/koala/outbound/mysql.go
@@ -7,8 +7,6 @@ import (
 	"github.com/v2pro/plz/countlog"
 )
 
-//var mysqlGreeting = []byte{53, 0, 0, 0, 10, 53, 46, 48, 46, 53, 49, 98, 0, 1, 0, 0, 0, 47, 85, 62, 116, 80, 114, 109, 75, 0, 12, 162, 33, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 76, 87, 84, 124, 52, 47, 46, 55, 107, 55, 110, 0}
-
 var mysqlGreeting []byte
 var maxAllowedPacketResp []byte
 

diff --git a/koala/outbound/outbound.go b/koala/outbound/outbound.go
@@ -77,15 +77,15 @@ func handleOutbound(conn *net.TCPConn) {
 		replayingSession := replaying.RetrieveTmp(*tcpAddr)
 		if replayingSession == nil {
 			if len(request) == 0 {
-				countlog.Warn("event!outbound.read request empty", "ctx", ctx, "i", i)
+				countlog.Warn("event!outbound.read can not find replaying session and request empty", "ctx", ctx, "i", i)
 				return
 			}
-			if protocol == "mysql" {
-				// when mysql connection setup at application startup
-				if err := applySimulation(simulateMysql, ctx, request, conn, nil); err != nil {
-					return
-				}
-			}
+			//if protocol == "mysql" {
+			//	// when mysql connection setup at application startup
+			//	if err := applySimulation(simulateMysql, ctx, request, conn, nil); err != nil {
+			//		return
+			//	}
+			//}
 			countlog.Error("event!outbound.outbound can not find replaying session",
 				"ctx", ctx, "addr", tcpAddr, "content", request, "protocol", protocol)
 			return

diff --git a/output/libs/koala-libc.so b/output/libs/koala-libc.so
diff --git a/output/libs/koala-replayer.so b/output/libs/koala-replayer.so
diff --git a/php/midi/res/replayer/koala-replayer.so b/php/midi/res/replayer/koala-replayer.so
diff --git a/php/midi/src/Midi/Differ/Differ.php b/php/midi/src/Midi/Differ/Differ.php
@@ -273,6 +273,10 @@ private function _getDiff(&$aBodyFst, $aBodySec)
             if (is_array($value) && is_array($aBodySec[$key])) {
                 $result = $this->_getDiff($value, $aBodySec[$key]);
                 if (!empty($result)) {
+                    if (isset($this->config['exclude-keys'][$key])) {
+                        $aBodyFst[$key] = '<fg=black;bg=cyan>' . $value . '</>';
+                        continue;
+                    }
                     $diff[$key] = $result;
                 }
                 continue;

diff --git a/php/midi/src/Midi/Koala/ParseRecorded.php b/php/midi/src/Midi/Koala/ParseRecorded.php
@@ -18,6 +18,7 @@
 use Midi\Koala\Replaying\ReplayingSession;
 use Midi\Koala\Replaying\ReturnInbound;
 use Midi\Parser\ParseFCGI;
+use Midi\Parser\ParseHTTP;
 use Midi\Reporter\Coverage;
 use Symfony\Component\Console\Output\OutputInterface;
 
@@ -177,22 +178,70 @@ private static function buildReturnInbound($recordedReturnInbound, $translatesFa
         return $returnInbound;
     }
 
+    /**
+     * fix http calloutbound
+     *
+     * one connection and multi actions:
+     *
+     * action1
+     *     request: GET /path/abc?...
+     *     response: (empty response)
+     * action2
+     *     request: (empty request)
+     *     response: abc
+     *
+     * fixed:
+     * action1
+     *     request: GET /path/abc?...
+     *     response: abc
+     */
     private static function buildActions($recordedSession)
     {
+        // for fix http CallOutbound
+        $outbounds = [];
+        $outboundIdx = 1;
+        $identifier2Idxs = [];
+        $fixOutboundsIdx = [];
+        $notFixIdentifiers = [];
+
         $ret = [];
         foreach ($recordedSession['Actions'] as $action) {
             switch ($action['ActionType']) {
                 case 'CallOutbound':
-                    $callOutbound = new CallOutbound();
-                    $callOutbound->setOccurredAt($action['OccurredAt']);
-                    $callOutbound->setActionIndex($action['ActionIndex']);
-                    $callOutbound->setActionType($action['ActionType']);
-                    $callOutbound['Peer'] = $action['Peer'];
-                    $callOutbound->setResponseTime($action['ResponseTime']);
-                    $callOutbound->setRequest(base64_encode(stripcslashes($action['Request'])));
-                    $callOutbound->setResponse(base64_encode(stripcslashes($action['Response'])));
-                    $callOutbound->setSocketFD($action['SocketFD']);
-                    $ret['CallOutbounds'][] = $callOutbound;
+                    $idx = $outboundIdx;
+                    $outbounds[$idx] = $action;
+                    ++$outboundIdx;
+
+                    $identifier = self::getIdentifier($action);
+                    if (empty($identifier)) {
+                        continue;
+                    }
+                    if (isset($identifier2Idxs[$identifier])) {
+                        $identifier2Idxs[$identifier][] = $idx;
+                    } else {
+                        $identifier2Idxs[$identifier] = [$idx];
+                    }
+
+                    if (isset($notFixIdentifiers[$identifier])) {
+                        continue;
+                    }
+                    if (empty($action['Request']) || empty($action['Response'])) {
+                        if (isset($fixOutboundsIdx[$identifier])) {
+                            $fixOutboundsIdx[$identifier][] = $idx;
+                        } else {
+                            // check first traffic is HTTP?
+                            if (isset($identifier2Idxs[$identifier])) {
+                                $firstIdx = $identifier2Idxs[$identifier][0];
+                            } else {
+                                $firstIdx = $idx;
+                            }
+                            if (!ParseHTTP::match($outbounds[$firstIdx]['Request'], $outbounds[$firstIdx]['Response'])) {
+                                $notFixIdentifiers[$identifier] = 1;
+                                continue;
+                            }
+                            $fixOutboundsIdx[$identifier] = [$idx];
+                        }
+                    }
                     break;
                 case 'AppendFile':
                     $ret['AppendFiles'][] = $action;
@@ -206,6 +255,36 @@ private static function buildActions($recordedSession)
             }
         }
 
+        foreach ($fixOutboundsIdx as $identifier => $todoFixIdxs) {
+            $c = count($todoFixIdxs);
+            for ($i = 0; $i < $c;) {
+                $todoFixIdx = $todoFixIdxs[$i];
+                if (empty($outbounds[$todoFixIdx]['Response']) && isset($todoFixIdxs[$i + 1])) {
+                    $nextIdx = $todoFixIdxs[$i + 1];
+                    if (empty($outbounds[$nextIdx]["Request"])) {
+                        $outbounds[$todoFixIdx]['Response'] = $outbounds[$nextIdx]["Response"];
+                        unset($outbounds[$nextIdx]);
+                        $i += 2;
+                        continue;
+                    }
+                }
+                ++$i;
+            }
+        }
+
+        foreach ($outbounds as $idx => $action) {
+            $callOutbound = new CallOutbound();
+            $callOutbound->setOccurredAt($action['OccurredAt']);
+            $callOutbound->setActionIndex($action['ActionIndex']);
+            $callOutbound->setActionType($action['ActionType']);
+            $callOutbound['Peer'] = $action['Peer'];
+            $callOutbound->setResponseTime($action['ResponseTime']);
+            $callOutbound->setRequest(base64_encode(stripcslashes($action['Request'])));
+            $callOutbound->setResponse(base64_encode(stripcslashes($action['Response'])));
+            $callOutbound->setSocketFD($action['SocketFD']);
+            $ret['CallOutbounds'][] = $callOutbound;
+        }
+
         return $ret;
     }
 
@@ -231,4 +310,12 @@ public static function uriAddXdebugArgs($requestUrl, &$onlyUri, $enableXdebug)
 
         return $requestUrl;
     }
+
+    private static function getIdentifier($action) {
+        if (empty($action['Peer'])) {
+            return '';
+        }
+        $peer = $action['Peer'];
+        return sprintf("%s:%d#%d", $peer['IP'], $peer['PORT'], $action['SocketFD']);
+    }
 }
diff --git a/php/midi/src/Midi/Parser/ParseHTTP.php b/php/midi/src/Midi/Parser/ParseHTTP.php
@@ -7,8 +7,8 @@ class ParseHTTP implements ParseInterface
 
     public static function match($request, $response = null)
     {
-        return preg_match('/^(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|CONNECT).*HTTP\/1\.[1|0]/', $request)
-            || preg_match('/^HTTP\/1.[0|1][[:space:]][1-5]\d\d/', $response);
+        return preg_match('/^(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|CONNECT)\s+.*\s+HTTP\/1\.[10]/', $request)
+            || preg_match('/^HTTP\/1.[01][[:space:]][1-5]\d\d/', $response);
     }
 
     public static function parse($request, $response = null)