From 9a126103fc80ff484d45b9e0040985efa7d914d3 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Sun, 20 Oct 2013 20:53:09 +0100 Subject: [PATCH 01/10] Ensure that TravisCI tests all subpackages Previously this was only running the integration tests in the root directory. `go test` needs to be given wildcard in order to pickup tests in sub directories/packages. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5588756b..c12d54c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.1 -script: sudo -E bash -c "source /etc/profile && gvm use go1.1 && export GOPATH=$HOME/gopath:$GOPATH && go test -v" +script: sudo -E bash -c "source /etc/profile && gvm use go1.1 && export GOPATH=$HOME/gopath:$GOPATH && go test -v ./..." From 9e175434d6f804da8931972c8f80e9582e24f62b Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Sun, 20 Oct 2013 20:58:01 +0100 Subject: [PATCH 02/10] Fix failing Elasticsearch test Use the local instance of `ReplaySettings` rather than `Settings` which is initialised as an empty struct and then populated by flag. Since we're not calling flag, this remains empty, and `esp.Init()` is passed an empty string. For the same reason the call to `settings.Parse()` is actually redundant. `Parse()` could be changed to use `r.ElastiSearchURI`, but it results in the plugin begin setup/registered twice, because it doesn't give us access to the host/port/index properties. --- replay/settings_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/replay/settings_test.go b/replay/settings_test.go index 373880bd..82e3fa69 100644 --- a/replay/settings_test.go +++ b/replay/settings_test.go @@ -49,10 +49,13 @@ func TestElasticSearchSettings(t *testing.T) { ElastiSearchURI: "host:10/index_name", } + // FIXME: This is redundant. We could assign `Settings = *settings` to + // check the code path in Init(), but it would result in the ES plugin + // being registered twice. settings.Parse() esp := &ESPlugin{} - esp.Init(Settings.ElastiSearchURI) + esp.Init(settings.ElastiSearchURI) if esp.ApiPort != "10" { t.Error("Port not match") From f4af3a9bd45582a0b3860f12064815c03d539262 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Fri, 18 Oct 2013 13:22:56 +0100 Subject: [PATCH 03/10] Allow injection of additional headers in replay Similar to curl(1) and siege(1): - Can be specified multiple times. - Will overwrite headers of the same key name in the original request. - An argument without a value (`Foo:`) will be set as empty. - Leading and trailing whitespace will be stripped from the key and value, otherwise it's difficult to determine where the key or value begin/end. --- README.md | 12 ++++++++++++ replay/request_factory.go | 4 ++++ replay/settings.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/README.md b/README.md index 394c38ed..b39ccd49 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,18 @@ gor replay -f "http://staging.server" -file requests.gor **Note:** Replay will preserve the original time differences between requests. +### Injecting headers +Additional headers can be injected/overwritten into requests during replay. +This may be useful if the hostname that staging responds to differs from +production, you need to identify requests generated by Gor, or enable +feature flagged functionality in an application: +``` +gor replay -f "http://staging.server" \ + -header "Host: staging.server" \ + -header "User-Agent: Replayed by Gor" -header " \ + -header "Enable-Feature-X: true" +``` + ## Stats diff --git a/replay/request_factory.go b/replay/request_factory.go index e3470dfe..3c8ecb35 100644 --- a/replay/request_factory.go +++ b/replay/request_factory.go @@ -75,6 +75,10 @@ func (f *RequestFactory) sendRequest(host *ForwardHost, requestBytes []byte) { request.RequestURI = "" request.URL, _ = url.ParseRequestURI(URL) + for _, header := range Settings.AdditionalHeaders { + request.Header.Set(header.Name, header.Value) + } + Debug("Sending request:", host.Url, request) tstart := time.Now() diff --git a/replay/settings.go b/replay/settings.go index 732bf115..86b3f8c2 100644 --- a/replay/settings.go +++ b/replay/settings.go @@ -1,6 +1,8 @@ package replay import ( + "errors" + "fmt" "flag" "os" "strconv" @@ -19,6 +21,31 @@ type ForwardHost struct { Stat *RequestStat } +type Headers []Header +type Header struct { + Name string + Value string +} + +func (h *Headers) String() string { + return fmt.Sprint(*h) +} + +func (h *Headers) Set(value string) error { + v := strings.SplitN(value, ":", 2) + if len(v) != 2 { + return errors.New("Expected `Key: Value`") + } + + header := Header{ + strings.TrimSpace(v[0]), + strings.TrimSpace(v[1]), + } + + *h = append(*h, header) + return nil +} + // ReplaySettings ListenerSettings contain all the needed configuration for setting up the replay type ReplaySettings struct { Port int @@ -34,6 +61,8 @@ type ReplaySettings struct { ElastiSearchURI string + AdditionalHeaders Headers + ResponseAnalyzePlugins []ResponseAnalyzer } @@ -106,4 +135,6 @@ func init() { flag.BoolVar(&Settings.Verbose, "verbose", false, "Log requests") flag.StringVar(&Settings.ElastiSearchURI, "es", "", "enable elasticsearch\n\tformat: hostname:9200/index_name") + + flag.Var(&Settings.AdditionalHeaders, "header", "Additional `Key: Value` header to inject/overwrite\n\tLeading and trailing whitespace will be stripped from the key and value\n\tMay be specified multiple times") } From d54da99667daae5efcbb103e62879d421057d0ef Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Sun, 20 Oct 2013 13:54:20 +0100 Subject: [PATCH 04/10] Unit test the parsing of -header arguments Per the features described in the preceding commit. --- replay/settings_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/replay/settings_test.go b/replay/settings_test.go index 82e3fa69..c8916a9f 100644 --- a/replay/settings_test.go +++ b/replay/settings_test.go @@ -1,6 +1,8 @@ package replay import ( + "flag" + "reflect" "testing" ) @@ -69,3 +71,41 @@ func TestElasticSearchSettings(t *testing.T) { t.Error("Index not match") } } + +func TestAdditionalHeaders(t *testing.T) { + args := []string{ + "-header", "Empty:", + "-header", "Foo: contains:multiple:colons", + "-header", "Host:nospaces.example.com", + "-header", "Authorization: Basic Zm9vOmJhcg==", + "-header", " User-Agent : Contains leading and trailing space ", + } + out := Headers{ + {"Empty", ""}, + {"Foo", "contains:multiple:colons"}, + {"Host", "nospaces.example.com"}, + {"Authorization", "Basic Zm9vOmJhcg=="}, + {"User-Agent", "Contains leading and trailing space"}, + } + + headers := Headers{} + fs := flag.NewFlagSet("TestHeaders", flag.ExitOnError) + fs.Var(&headers, "header", "blah") + fs.Parse(args) + + if !reflect.DeepEqual(headers, out) { + t.Error("Headers not parsed as expected") + } + + args = []string{ + "-header", "contains no colons", + } + headers = Headers{} + fs = flag.NewFlagSet("TestHeaders", flag.ContinueOnError) + fs.Var(&headers, "header", "blah") + err := fs.Parse(args) + + if err == nil { + t.Error("Invalid header should be rejected") + } +} From 9c6802f5e8931988f2f2f996a550f38caa725446 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Mon, 21 Oct 2013 12:02:36 +0100 Subject: [PATCH 05/10] Integration test for header injection Re-using one of the existing integration tests. If we were to refactor some of the logic to send and receieve a single request then it would make sense to split this, and the cookie validation, out to separate test cases. --- integration_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/integration_test.go b/integration_test.go index 51ce7b99..5b8e4b40 100644 --- a/integration_test.go +++ b/integration_test.go @@ -32,6 +32,8 @@ type Env struct { ReplayLimit int ListenerLimit int ForwardPort int + + AdditionalHeaders replay.Headers } func (e *Env) start() (p int) { @@ -79,6 +81,10 @@ func (e *Env) startReplay(port int, forwardPort int) { replay.Settings.ForwardAddress += "|" + strconv.Itoa(e.ReplayLimit) } + if len(e.AdditionalHeaders) > 0 { + replay.Settings.AdditionalHeaders = e.AdditionalHeaders + } + replay.Settings.ForwardAddress += ",127.0.0.1:" + strconv.Itoa(forwardPort+1) replay.Run() @@ -131,6 +137,9 @@ func TestReplay(t *testing.T) { replayHandler := func(w http.ResponseWriter, r *http.Request) { isEqual(t, r.URL.Path, request.URL.Path) + isEqual(t, r.Header.Get("New-Header"), "Inserted") + isEqual(t, r.Header.Get("X-Forwarded-Proto"), "Overwritten") + if len(r.Cookies()) > 0 { isEqual(t, r.Cookies()[0].Value, request.Cookies()[0].Value) } else { @@ -150,6 +159,10 @@ func TestReplay(t *testing.T) { Verbose: true, ListenHandler: listenHandler, ReplayHandler: replayHandler, + AdditionalHeaders: replay.Headers{ + {"New-Header", "Inserted"}, + {"X-Forwarded-Proto", "Overwritten"}, + }, } p := env.start() From 714307890ecbb46efbe352ee3e897971bf86a843 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Tue, 22 Oct 2013 14:03:39 +0100 Subject: [PATCH 06/10] [#47 #36] Serialize requests to file using gob To replace the serial plaintext representation of request and timestamp. This fixes a bug whereby we were unable to delimit POST requests with bodies because they didn't end in `\r\n\r\n`, frequently resulting in: ``` --- FAIL: TestSavingRequestToFileAndReplayThem (2.80 seconds) integration_test.go:346: Timeout error ``` Now we don't need to play guesswork with `Content-Length` headers or introduce a custom delimiter. As a bonus, the code required is also slightly simpler. It is no longer possible to manipulate the `.gor` file by hand; but it does make it trivial to write a tool that can parse and modify. There are some `go fmt` indent changes rolled into `replay_file_parser`. --- listener/listener.go | 29 ++++++------- replay/replay_file_parser.go | 82 +++++++++++++----------------------- 2 files changed, 43 insertions(+), 68 deletions(-) diff --git a/listener/listener.go b/listener/listener.go index 14bb9513..9965aa99 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -7,6 +7,7 @@ package listener import ( "bufio" "bytes" + "encoding/gob" "fmt" "log" "net" @@ -16,6 +17,11 @@ import ( "time" ) +type ParsedRequest struct { + Timestamp int64 + Request []byte +} + // Debug enables logging only if "--verbose" flag passed func Debug(v ...interface{}) { if Settings.Verbose { @@ -49,7 +55,7 @@ func Run() { fmt.Println("Listening for HTTP traffic on", Settings.Address+":"+strconv.Itoa(Settings.Port)) - var messageLogger *log.Logger + var fileEnc *gob.Encoder if Settings.FileToReplayPath != "" { @@ -60,13 +66,10 @@ func Run() { log.Fatal("Cannot open file %q. Error: %s", Settings.FileToReplayPath, err) } - messageLogger = log.New(file, "", 0) - } - - if messageLogger == nil { - fmt.Println("Forwarding requests to replay server:", Settings.ReplayAddress, "Limit:", Settings.ReplayLimit) - } else { + fileEnc = gob.NewEncoder(file) fmt.Println("Saving requests to file", Settings.FileToReplayPath) + } else { + fmt.Println("Forwarding requests to replay server:", Settings.ReplayAddress, "Limit:", Settings.ReplayLimit) } // Sniffing traffic from given address @@ -92,16 +95,10 @@ func Run() { currentRPS++ } - if messageLogger != nil { + if Settings.FileToReplayPath != "" { go func() { - messageBuffer := new(bytes.Buffer) - messageWriter := bufio.NewWriter(messageBuffer) - - fmt.Fprintf(messageWriter, "%v\n", time.Now().UnixNano()) - fmt.Fprintf(messageWriter, "%s", string(m.Bytes())) - - messageWriter.Flush() - messageLogger.Println(messageBuffer.String()) + message := ParsedRequest{time.Now().UnixNano(), m.Bytes()} + fileEnc.Encode(message) }() } else { go sendMessage(m) diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index d1854003..216d5d24 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -1,79 +1,57 @@ package replay import ( - "bufio" - "log" - "os" - "bytes" - "strconv" + "bytes" + "encoding/gob" + "io" + "io/ioutil" + "log" - "fmt" + "fmt" ) type ParsedRequest struct { - Request []byte - Timestamp int64 + Timestamp int64 + Request []byte } func (self ParsedRequest) String() string { - return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) + return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) } func parseReplayFile() (requests []ParsedRequest, err error) { - requests, err = readLines(Settings.FileToReplayPath) + requests, err = readLines(Settings.FileToReplayPath) - if err != nil { - log.Fatalf("readLines: %s", err) - } + if err != nil { + log.Fatalf("readLines: %s", err) + } - return + return } // readLines reads a whole file into memory -// and returns a slice of its lines. +// and returns a slice of request+timestamps. func readLines(path string) (requests []ParsedRequest, err error) { - file, err := os.Open(path) - - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - scanner.Split(scanLinesFunc) - - for scanner.Scan() { - if len(scanner.Text()) > 5 { - buf := append([]byte(nil), scanner.Bytes()...) - i := bytes.IndexByte(buf, '\n') - timestamp, _ := strconv.Atoi(string(buf[:i])) - pr := ParsedRequest{buf[i + 1:], int64(timestamp)} - - requests = append(requests, pr) - } - } + file, err := ioutil.ReadFile(path) - return requests, scanner.Err() -} - -// scanner spliting logic -func scanLinesFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil + if err != nil { + return nil, err } - delimiter := []byte{'\r', '\n', '\r', '\n', '\n'} + fileBuf := bytes.NewBuffer(file) + fileDec := gob.NewDecoder(fileBuf) - // We have a http request end: \r\n\r\n - if i := bytes.Index(data, delimiter); i >= 0 { - return (i + len(delimiter)), data[0:(i + len(delimiter))], nil - } + for err == nil { + var reqBuf ParsedRequest + err = fileDec.Decode(&reqBuf) + + if err == io.EOF { + err = nil + break + } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil + requests = append(requests, reqBuf) } - // Request more data. - return 0, nil, nil + return requests, err } From e7c1d86bfacff37099ca3797472c54d2bfdec679 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Wed, 23 Oct 2013 17:06:58 +0100 Subject: [PATCH 07/10] Move ParsedRequest{} to gor/utils sub-package This de-dupes the type definition used in both `listener` and `replay`. --- listener/listener.go | 9 +++------ replay/replay_file_parser.go | 17 ++++------------- utils/utils.go | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 utils/utils.go diff --git a/listener/listener.go b/listener/listener.go index 9965aa99..3427fa50 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -15,12 +15,9 @@ import ( "os" "strconv" "time" -) -type ParsedRequest struct { - Timestamp int64 - Request []byte -} + "github.com/buger/gor/utils" +) // Debug enables logging only if "--verbose" flag passed func Debug(v ...interface{}) { @@ -97,7 +94,7 @@ func Run() { if Settings.FileToReplayPath != "" { go func() { - message := ParsedRequest{time.Now().UnixNano(), m.Bytes()} + message := utils.ParsedRequest{time.Now().UnixNano(), m.Bytes()} fileEnc.Encode(message) }() } else { diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index 216d5d24..4cecf5fc 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -7,19 +7,10 @@ import ( "io/ioutil" "log" - "fmt" + "github.com/buger/gor/utils" ) -type ParsedRequest struct { - Timestamp int64 - Request []byte -} - -func (self ParsedRequest) String() string { - return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) -} - -func parseReplayFile() (requests []ParsedRequest, err error) { +func parseReplayFile() (requests []utils.ParsedRequest, err error) { requests, err = readLines(Settings.FileToReplayPath) if err != nil { @@ -31,7 +22,7 @@ func parseReplayFile() (requests []ParsedRequest, err error) { // readLines reads a whole file into memory // and returns a slice of request+timestamps. -func readLines(path string) (requests []ParsedRequest, err error) { +func readLines(path string) (requests []utils.ParsedRequest, err error) { file, err := ioutil.ReadFile(path) if err != nil { @@ -42,7 +33,7 @@ func readLines(path string) (requests []ParsedRequest, err error) { fileDec := gob.NewDecoder(fileBuf) for err == nil { - var reqBuf ParsedRequest + var reqBuf utils.ParsedRequest err = fileDec.Decode(&reqBuf) if err == io.EOF { diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 00000000..c1789096 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,14 @@ +package utils + +import ( + "fmt" +) + +type ParsedRequest struct { + Timestamp int64 + Request []byte +} + +func (self ParsedRequest) String() string { + return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) +} From 2f049dddbd2b6b94c975aa247734107d65fd670a Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Wed, 23 Oct 2013 21:07:35 +0100 Subject: [PATCH 08/10] Rename ParsedRequest to RawRequest Since it contains a raw byte slice of the request data rather than a `http.Request` object. --- listener/listener.go | 2 +- replay/replay_file_parser.go | 6 +++--- utils/utils.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/listener/listener.go b/listener/listener.go index 3427fa50..5239388b 100644 --- a/listener/listener.go +++ b/listener/listener.go @@ -94,7 +94,7 @@ func Run() { if Settings.FileToReplayPath != "" { go func() { - message := utils.ParsedRequest{time.Now().UnixNano(), m.Bytes()} + message := utils.RawRequest{time.Now().UnixNano(), m.Bytes()} fileEnc.Encode(message) }() } else { diff --git a/replay/replay_file_parser.go b/replay/replay_file_parser.go index 4cecf5fc..97197f72 100644 --- a/replay/replay_file_parser.go +++ b/replay/replay_file_parser.go @@ -10,7 +10,7 @@ import ( "github.com/buger/gor/utils" ) -func parseReplayFile() (requests []utils.ParsedRequest, err error) { +func parseReplayFile() (requests []utils.RawRequest, err error) { requests, err = readLines(Settings.FileToReplayPath) if err != nil { @@ -22,7 +22,7 @@ func parseReplayFile() (requests []utils.ParsedRequest, err error) { // readLines reads a whole file into memory // and returns a slice of request+timestamps. -func readLines(path string) (requests []utils.ParsedRequest, err error) { +func readLines(path string) (requests []utils.RawRequest, err error) { file, err := ioutil.ReadFile(path) if err != nil { @@ -33,7 +33,7 @@ func readLines(path string) (requests []utils.ParsedRequest, err error) { fileDec := gob.NewDecoder(fileBuf) for err == nil { - var reqBuf utils.ParsedRequest + var reqBuf utils.RawRequest err = fileDec.Decode(&reqBuf) if err == io.EOF { diff --git a/utils/utils.go b/utils/utils.go index c1789096..5ca5cd7d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,11 +4,11 @@ import ( "fmt" ) -type ParsedRequest struct { +type RawRequest struct { Timestamp int64 Request []byte } -func (self ParsedRequest) String() string { +func (self RawRequest) String() string { return fmt.Sprintf("Request: %v, timestamp: %v", string(self.Request), self.Timestamp) } From 24a240c819483e8ec825ffa5b2cb00b7f757b24a Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Thu, 24 Oct 2013 12:42:47 +0100 Subject: [PATCH 09/10] [README] Stylise GOV.UK link Use `https://www` to spare a redirect. And capped-up-cos-we-mean-business: http://www.theregister.co.uk/2012/02/01/gov_uk_single_domain_release/ --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b39ccd49..29eaafc7 100644 --- a/README.md +++ b/README.md @@ -147,5 +147,5 @@ More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-ope ## Companies using Gor * http://granify.com -* http://gov.uk ([Government Digital Service](http://digital.cabinetoffice.gov.uk/)) +* [GOV.UK](https://www.gov.uk) ([Government Digital Service](http://digital.cabinetoffice.gov.uk/)) * To add your company drop me a line to github.com/buger or leonsbox@gmail.com From bc4fabd0da71bd98169b96f4e292ca2809569fa8 Mon Sep 17 00:00:00 2001 From: Dan Carley Date: Thu, 24 Oct 2013 13:02:14 +0100 Subject: [PATCH 10/10] [#31] Document BasicAuth injection Has been asked about previously. `http.Client.Do` does this for us. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 29eaafc7..a00036ea 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,16 @@ gor replay -f "http://staging.server" \ -header "Enable-Feature-X: true" ``` +### Basic Auth +If your development or staging environment is protected by Basic Authentication +then those credentials can be injected in during the replay: +``` +gor replay -f "http://user1:pass1@dev.server,http://user2:pass2@staging.server" +``` + +**Note:** This will overwrite any `Authorization` headers in the original +request. + ## Stats