From 714307890ecbb46efbe352ee3e897971bf86a843 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Tue, 22 Oct 2013 14:03:39 +0100 Subject: [PATCH 1/3] [#47 #36] Serialize requests to file using gob To replace the serial plaintext representation of request and timestamp. This fixes a bug whereby we were unable to delimit POST requests with bodies because they didn't end in `\r\n\r\n`, frequently resulting in: ``` --- FAIL: TestSavingRequestToFileAndReplayThem (2.80 seconds) integration_test.go:346: Timeout error ``` Now we don't need to play guesswork with `Content-Length` headers or introduce a custom delimiter. As a bonus, the code required is also slightly simpler. It is no longer possible to manipulate the `.gor` file by hand; but it does make it trivial to write a tool that can parse and modify. There are some `go fmt` indent changes rolled into `replay_file_parser`. --- listener/listener.go | 29 ++++++------- replay/replay_file_parser.go | 82 +++++++++++++----------------------- 2 files changed, 43 insertions(+), 68 deletions(-) diff --git a/listener/listener.go b/listener/listener.go index 14bb9513..9965aa99 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -7,6 +7,7 @@ package listener import ( "bufio" "bytes" + "encoding/gob" "fmt" "log" "net" @@ -16,6 +17,11 @@ import ( "time" ) +type ParsedRequest struct { + Timestamp int64 + Request []byte +} + // Debug enables logging only if "--verbose" flag passed func Debug(v ...interface{}) { if Settings.Verbose { @@ -49,7 +55,7 @@ func Run() { fmt.Println("Listening for HTTP traffic on", Settings.Address+":"+strconv.Itoa(Settings.Port)) - var messageLogger *log.Logger + var fileEnc *gob.Encoder if Settings.FileToReplayPath != "" { @@ -60,13 +66,10 @@ func Run() { log.Fatal("Cannot open file %q. Error: %s", Settings.FileToReplayPath, err) } - messageLogger = log.New(file, "", 0) - } - - if messageLogger == nil { - fmt.Println("Forwarding requests to replay server:", Settings.ReplayAddress, "Limit:", Settings.ReplayLimit) - } else { + fileEnc = gob.NewEncoder(file) fmt.Println("Saving requests to file", Settings.FileToReplayPath) + } else { + fmt.Println("Forwarding requests to replay server:", Settings.ReplayAddress, "Limit:", Settings.ReplayLimit) } // Sniffing traffic from given address @@ -92,16 +95,10 @@ func Run() { currentRPS++ } - if messageLogger != nil { + if Settings.FileToReplayPath != "" { go func() { - messageBuffer := new(bytes.Buffer) - messageWriter := bufio.NewWriter(messageBuffer) - - fmt.Fprintf(messageWriter, "%v\n", time.Now().UnixNano()) - fmt.Fprintf(messageWriter, "%s", string(m.Bytes())) - - messageWriter.Flush() - messageLogger.Println(messageBuffer.String()) + message := ParsedRequest{time.Now().UnixNano(), m.Bytes()} + fileEnc.Encode(message) }() } else { go sendMessage(m) diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index d1854003..216d5d24 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -1,79 +1,57 @@ package replay import ( - "bufio" - "log" - "os" - "bytes" - "strconv" + "bytes" + "encoding/gob" + "io" + "io/ioutil" + "log" - "fmt" + "fmt" ) type ParsedRequest struct { - Request []byte - Timestamp int64 + Timestamp int64 + Request []byte } func (self ParsedRequest) String() string { - return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) + return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) } func parseReplayFile() (requests []ParsedRequest, err error) { - requests, err = readLines(Settings.FileToReplayPath) + requests, err = readLines(Settings.FileToReplayPath) - if err != nil { - log.Fatalf("readLines: %s", err) - } + if err != nil { + log.Fatalf("readLines: %s", err) + } - return + return } // readLines reads a whole file into memory -// and returns a slice of its lines. +// and returns a slice of request+timestamps. func readLines(path string) (requests []ParsedRequest, err error) { - file, err := os.Open(path) - - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - scanner.Split(scanLinesFunc) - - for scanner.Scan() { - if len(scanner.Text()) > 5 { - buf := append([]byte(nil), scanner.Bytes()...) - i := bytes.IndexByte(buf, '\n') - timestamp, _ := strconv.Atoi(string(buf[:i])) - pr := ParsedRequest{buf[i + 1:], int64(timestamp)} - - requests = append(requests, pr) - } - } + file, err := ioutil.ReadFile(path) - return requests, scanner.Err() -} - -// scanner spliting logic -func scanLinesFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil + if err != nil { + return nil, err } - delimiter := []byte{'\r', '\n', '\r', '\n', '\n'} + fileBuf := bytes.NewBuffer(file) + fileDec := gob.NewDecoder(fileBuf) - // We have a http request end: \r\n\r\n - if i := bytes.Index(data, delimiter); i >= 0 { - return (i + len(delimiter)), data[0:(i + len(delimiter))], nil - } + for err == nil { + var reqBuf ParsedRequest + err = fileDec.Decode(&reqBuf) + + if err == io.EOF { + err = nil + break + } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil + requests = append(requests, reqBuf) } - // Request more data. - return 0, nil, nil + return requests, err } From e7c1d86bfacff37099ca3797472c54d2bfdec679 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Wed, 23 Oct 2013 17:06:58 +0100 Subject: [PATCH 2/3] Move ParsedRequest{} to gor/utils sub-package This de-dupes the type definition used in both `listener` and `replay`. --- listener/listener.go | 9 +++------ replay/replay_file_parser.go | 17 ++++------------- utils/utils.go | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 utils/utils.go diff --git a/listener/listener.go b/listener/listener.go index 9965aa99..3427fa50 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -15,12 +15,9 @@ import ( "os" "strconv" "time" -) -type ParsedRequest struct { - Timestamp int64 - Request []byte -} + "github.com/buger/gor/utils" +) // Debug enables logging only if "--verbose" flag passed func Debug(v ...interface{}) { @@ -97,7 +94,7 @@ func Run() { if Settings.FileToReplayPath != "" { go func() { - message := ParsedRequest{time.Now().UnixNano(), m.Bytes()} + message := utils.ParsedRequest{time.Now().UnixNano(), m.Bytes()} fileEnc.Encode(message) }() } else { diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index 216d5d24..4cecf5fc 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -7,19 +7,10 @@ import ( "io/ioutil" "log" - "fmt" + "github.com/buger/gor/utils" ) -type ParsedRequest struct { - Timestamp int64 - Request []byte -} - -func (self ParsedRequest) String() string { - return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) -} - -func parseReplayFile() (requests []ParsedRequest, err error) { +func parseReplayFile() (requests []utils.ParsedRequest, err error) { requests, err = readLines(Settings.FileToReplayPath) if err != nil { @@ -31,7 +22,7 @@ func parseReplayFile() (requests []ParsedRequest, err error) { // readLines reads a whole file into memory // and returns a slice of request+timestamps. -func readLines(path string) (requests []ParsedRequest, err error) { +func readLines(path string) (requests []utils.ParsedRequest, err error) { file, err := ioutil.ReadFile(path) if err != nil { @@ -42,7 +33,7 @@ func readLines(path string) (requests []ParsedRequest, err error) { fileDec := gob.NewDecoder(fileBuf) for err == nil { - var reqBuf ParsedRequest + var reqBuf utils.ParsedRequest err = fileDec.Decode(&reqBuf) if err == io.EOF { diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 00000000..c1789096 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,14 @@ +package utils + +import ( + "fmt" +) + +type ParsedRequest struct { + Timestamp int64 + Request []byte +} + +func (self ParsedRequest) String() string { + return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) +} From 2f049dddbd2b6b94c975aa247734107d65fd670a Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Wed, 23 Oct 2013 21:07:35 +0100 Subject: [PATCH 3/3] Rename ParsedRequest to RawRequest Since it contains a raw byte slice of the request data rather than a `http.Request` object. --- listener/listener.go | 2 +- replay/replay_file_parser.go | 6 +++--- utils/utils.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/listener/listener.go b/listener/listener.go index 3427fa50..5239388b 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -94,7 +94,7 @@ func Run() { if Settings.FileToReplayPath != "" { go func() { - message := utils.ParsedRequest{time.Now().UnixNano(), m.Bytes()} + message := utils.RawRequest{time.Now().UnixNano(), m.Bytes()} fileEnc.Encode(message) }() } else { diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index 4cecf5fc..97197f72 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -10,7 +10,7 @@ import ( "github.com/buger/gor/utils" ) -func parseReplayFile() (requests []utils.ParsedRequest, err error) { +func parseReplayFile() (requests []utils.RawRequest, err error) { requests, err = readLines(Settings.FileToReplayPath) if err != nil { @@ -22,7 +22,7 @@ func parseReplayFile() (requests []utils.ParsedRequest, err error) { // readLines reads a whole file into memory // and returns a slice of request+timestamps. -func readLines(path string) (requests []utils.ParsedRequest, err error) { +func readLines(path string) (requests []utils.RawRequest, err error) { file, err := ioutil.ReadFile(path) if err != nil { @@ -33,7 +33,7 @@ func readLines(path string) (requests []utils.ParsedRequest, err error) { fileDec := gob.NewDecoder(fileBuf) for err == nil { - var reqBuf utils.ParsedRequest + var reqBuf utils.RawRequest err = fileDec.Decode(&reqBuf) if err == io.EOF { diff --git a/utils/utils.go b/utils/utils.go index c1789096..5ca5cd7d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,11 +4,11 @@ import ( "fmt" ) -type ParsedRequest struct { +type RawRequest struct { Timestamp int64 Request []byte } -func (self ParsedRequest) String() string { +func (self RawRequest) String() string { return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) }