Merge pull request #46 from buger/module_architecture

Merge listener and replay functionality

.travis.yml

@@ -1,3 +1,3 @@
 language: go 
 go: 1.1
-script: sudo -E bash -c "source /etc/profile && gvm use go1.1 && export GOPATH=$HOME/gopath:$GOPATH && go test -v ./..."
+script: sudo -E bash -c "source /etc/profile && gvm use go1.1 && export GOPATH=$HOME/gopath:$GOPATH && go test -v"

CHANGELOG.md

@@ -1,3 +1,10 @@
+v0.7.0 - 31 Oct 2013
+* New modular architecture. Listener and Replay functionality merged.
+* Added option to equally split traffic between multiple outputs: --split-output true
+* Saving requests to file and replaying from it
+* Injecting custom headers to http requests
+* Advanced stats using ElasticSearch
+
 v0.3.5 - 15 Sep 2013
 * Significantly improved test coverage
 * Fixed bug with redirect replay https://github.com/buger/gor/pull/15

ELASTICSEARCH.md

@@ -39,22 +39,12 @@ gor
 Start your gor replay server with elasticsearch option:
 
 ```
-./gor replay -f <your-dev-system-url> -ip <your_replay_listener_ip> -p <your_replay_listener_port> -es <elasticsearch_host>:<elasticsearch_port>/<elasticsearch_index>
+./gor --input-raw :8000 --output-http http://staging.com  --output-http-elasticsearch localhost:9200/gor
 ```
 
-In our example this would be:
-
-```
-./gor replay -f <your-dev-system-url> -ip <your_replay_listener_ip> -p 28020 -es localhost:9200/gor
-```
 
 (You don't have to create the index upfront. That will be done for you automatically)
 
-Now start your gor listen process as usual:
-
-```
-sudo gor listen -p 80 -r replay.server.local:28020
-```
 
 Now visit your kibana url, load the predefined dashboard from the gist https://gist.github.com/gottwald/b2c875037f24719a9616 and watch the data rush in.
 

Procfile

@@ -1,4 +1,4 @@
 web: python -m SimpleHTTPServer 8000
 replayed_web: python -m SimpleHTTPServer 8001
-listener: sudo -E go run gor.go listen -p 8000 -r localhost:8002 --verbose
-replay: go run gor.go replay -f localhost:8001 -p 8002 --verbose
+listener: sudo -E go run ./bin/gor.go --input-raw :8000 --output-tcp :8002 --verbose
+replay: go run ./bin/gor.go --input-tcp :8002 --output-http localhost:8001 --verbose

README.md

@@ -9,116 +9,110 @@ Its main goal is to replay traffic from production servers to staging and dev en
 Now you can test your code on real user sessions in an automated and repeatable fashion.
 **No more falling down in production!**
 
-Gor consists of 2 parts: listener and replay servers.
-
-The listener server catches http traffic from a given port in real-time
-and sends it to the replay server or saves to file.
-The replay server forwards traffic to a given address.
+Here is basic worlkflow: The listener server catches http traffic and sends it to the replay server or saves to file.The replay server forwards traffic to a given address.
 
 
 ![Diagram](http://i.imgur.com/9mqj2SK.png)
 
 
-## Basic example
+## Examples
 
+### Capture traffic from port
 ```bash
 # Run on servers where you want to catch traffic. You can run it on each `web` machine.
-sudo gor listen -p 80 -r replay.server.local:28020
+sudo gor --input-raw :80 --output-tcp replay.local:28020
+
+# Replay server (replay.local).
+gor --input-tcp replay.local:28020 --output-http http://staging.com
+```
 
-# Replay server (replay.server.local).
-gor replay -f http://staging.server -p 28020
+### Using 1 Gor instance for both listening and replaying
+It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
+
+```
+sudo gor --input-raw :80 --output-http "http://staging.com"
 ```
 
 ## Advanced use
 
 ### Rate limiting
-Both replay and listener supports rate limiting. It can be useful if you want
+Both replay and listener support rate limiting. It can be useful if you want
 forward only part of production traffic and not overload your staging
 environment. You can specify your desired requests per second using the
 "|" operator after the server address:
 
+#### Limiting replay
 ```
 # staging.server will not get more than 10 requests per second
-gor replay -f "http://staging.server|10"
+gor --input-tcp :28020 --output-http "http://staging.com|10"
 ```
 
+#### Limiting listener
 ```
 # replay server will not get more than 10 requests per second
 # useful for high-load environments
-gor listen -p 8080 -r "replay.server.local:28020|10"
+gor --input-raw :80 --output-tcp "replay.local:28020|10"
 ```
 
 ### Forward to multiple addresses
 
-You can forward traffic to multiple endpoints. Just separate the addresses by comma.
+You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
 ```
-gor replay -f "http://staging.server|10,http://dev.server|5"
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
 ```
 
-### Saving requests to file
-You can save requests to file:
+#### Splitting traffic
+By default it will send same traffic to all outputs, but you have options to equally split it:
+
 ```
-gor listen -p 8080 -file requests.gor
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
 ```
 
-And replay them later:
+### Saving requests to file
+You can save requests to file, and replay them later:
 ```
-gor replay -f "http://staging.server" -file requests.gor
+# write to file
+gor --input-raw :80 --output-file requests.gor
+
+# read from file
+gor --input-file requests.gor --output-http "http://staging.com"
 ```
 
 **Note:** Replay will preserve the original time differences between requests.
 
 ### Injecting headers
-Additional headers can be injected/overwritten into requests during replay.
-This may be useful if the hostname that staging responds to differs from
-production, you need to identify requests generated by Gor, or enable
-feature flagged functionality in an application:
+
+Additional headers can be injected/overwritten into requests during replay. This may be useful if the hostname that staging responds to differs from production, you need to identify requests generated by Gor, or enable feature flagged functionality in an application:
+
 ```
-gor replay -f "http://staging.server" \
-    -header "Host: staging.server" \
-    -header "User-Agent: Replayed by Gor" -header " \
-    -header "Enable-Feature-X: true"
+gor --input-raw :80 --output-http "http://staging.server" \
+    --output-http-header "Host: staging.server" \
+    --output-http-header "User-Agent: Replayed by Gor" -header " \
+    --output-http-header "Enable-Feature-X: true"
 ```
 
 ### Basic Auth
-If your development or staging environment is protected by Basic Authentication
-then those credentials can be injected in during the replay:
+
+If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
+
 ```
-gor replay -f "http://user1:[email protected],http://user2:pass2@staging.server"
+gor --input-raw :80 --output-http "http://user:pass@staging .com"
 ```
 
-**Note:** This will overwrite any `Authorization` headers in the original
-request.
+Note: This will overwrite any Authorization headers in the original request.
 
 ## Stats 
-
-
 ### ElasticSearch 
-For deep reponse analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
+For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
 
 ```
-gor replay -f "http://staging.server" -es "es_host:api_port/index_name"
+gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
 ```
 
 
 ## Additional help
-```
-$ gor listen -h
-Usage of ./bin/gor-linux:
-  -i="any": By default it try to listen on all network interfaces.To get list of interfaces run `ifconfig`
-  -p=80: Specify the http server port whose traffic you want to capture
-  -r="localhost:28020": Address of replay server.
-```
 
-```
-$ gor replay -h
-Usage of ./bin/gor-linux:
-  -f="http://localhost:8080": http address to forward traffic.
-	You can limit requests per second by adding `|#{num}` after address.
-	If you have multiple addresses with different limits. For example: http://staging.example.com|100,http://dev.example.com|10
-  -ip="0.0.0.0": ip addresses to listen on
-  -p=28020: specify port number
-```
+Feel free to ask question directly by email or by creating github issue.
 
 ## Latest releases (including binaries)
 
@@ -128,14 +122,14 @@ https://github.com/buger/gor/releases
 1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set.
 2. `go get github.com/buger/gor`.
 3. `cd $GOPATH/src/github.com/buger/gor`
-4. `go build gor.go` to get binary, or `go run gor.go` to build and run (useful for development)
+4. `go build ./bin/gor.go` to get binary, or `go run ./bin/gor.go` to build and run (useful for development)
 
 ## FAQ
 
 ### What OS are supported?
 For now only Linux based. *BSD (including MacOS is not supported yet, check https://github.com/buger/gor/issues/22 for details)
 
-### Why does the `gor listener` requires sudo or root access?
+### Why does the `--input-raw` requires sudo or root access?
 Listener works by sniffing traffic from a given port. It's accessible
 only by using sudo or root access.
 

gor.go → bin/gor.go

@@ -1,24 +1,17 @@
 // Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
 // Now you can test your code on real user sessions in an automated and repeatable fashion.
-//
-// Gor consists of 2 parts: listener and replay servers.
-// Listener catch http traffic from given port in real-time and send it to replay server via UDP. Replay server forwards traffic to given address.
 package main
 
 import (
 	"flag"
 	"fmt"
 	"log"
 	"os"
+	"runtime/debug"
 	"runtime/pprof"
 	"time"
 
-	"github.com/buger/gor/listener"
-	"github.com/buger/gor/replay"
-)
-
-const (
-	VERSION = "0.3.5"
+	"github.com/buger/gor"
 )
 
 var (
@@ -28,32 +21,38 @@ var (
 )
 
 func main() {
+	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
 			if _, ok := r.(error); !ok {
-				fmt.Errorf("pkg: %v", r)
+				fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
 			}
 		}
 	}()
 
-	fmt.Println("Version:", VERSION)
+	fmt.Println("Version:", gor.VERSION)
+
+	flag.Parse()
+	gor.InitPlugins()
 
-	if len(os.Args) > 1 {
-		mode = os.Args[1]
+	if len(gor.Plugins.Inputs) == 0 || len(gor.Plugins.Outputs) == 0 {
+		log.Fatal("Required at least 1 input and 1 output")
 	}
 
-	if mode != "listen" && mode != "replay" {
-		fmt.Println("Usage: \n\tgor listen -h\n\tgor replay -h")
-		return
+	if *memprofile != "" {
+		profileMEM(*memprofile)
 	}
 
-	// Remove mode attr
-	os.Args = append(os.Args[:1], os.Args[2:]...)
+	if *cpuprofile != "" {
+		profileCPU(*cpuprofile)
+	}
 
-	flag.Parse()
+	gor.Start(nil)
+}
 
-	if *cpuprofile != "" {
-		f, err := os.Create(*cpuprofile)
+func profileCPU(cpuprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -65,9 +64,11 @@ func main() {
 			log.Println("Stop profiling after 60 seconds")
 		})
 	}
+}
 
-	if *memprofile != "" {
-		f, err := os.Create(*memprofile)
+func profileMEM(memprofile string) {
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -76,11 +77,4 @@ func main() {
 			f.Close()
 		})
 	}
-
-	switch mode {
-	case "listen":
-		listener.Run()
-	case "replay":
-		replay.Run()
-	}
 }