From c66e686f6542056542dfb32debe58e5ecd406781 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 11 Dec 2018 11:46:06 -0500 Subject: [PATCH] [server/kit] adding NewLogger to use for background logging + updated docs (#170) * adding NewLogger to use for background logging + upped kit docs * doc cleanup * removing tab from docs * p->P --- README.md | 19 +++++++++---------- server/kit/README.md | 23 +++++++++++++++-------- server/kit/gae_log.go | 12 ++++++++++++ server/kit/kitserver.go | 33 ++++++++++++--------------------- server/kit/log.go | 19 +++++++++++++++++++ server/kit/server.go | 5 ----- server/kit/server_gae.go | 11 ----------- 7 files changed, 67 insertions(+), 55 deletions(-) delete mode 100644 server/kit/server_gae.go diff --git a/README.md b/README.md index 9e0170de5..41595108c 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,15 @@ It offers 2 server implementations: #### [`server/kit`](https://godoc.org/github.com/NYTimes/gizmo/server/kit) -This is an experimental package in Gizmo! - -* The rationale behind this package: - * A more opinionated server with fewer choices - * go-kit is used for serving HTTP/JSON & gRPC is used for serving HTTP2/RPC - * Monitoring and metrics are handled by a sidecar (ie. Cloud Endpoints) - * Logs always go to stdout/stderr - * Using Go's 1.8 graceful HTTP shutdown - * Services using this package are meant for deploy to GCP with GKE and Cloud Endpoints. - +The `server/kit` package embodies Gizmo's goals to combine with go-kit. + +* In this package you'll find: + * A more opinionated server with fewer choices. + * go-kit used for serving HTTP/JSON & gRPC used for serving HTTP2/RPC + * Monitoring and metrics are automatically registered if running within App Engine. + * Logs go to stdout locally or directly to Stackdriver when in GCP. + * Using Go's 1.8 graceful HTTP shutdown. + * Services using this package are expected to deploy to GCP. #### [`auth`](https://godoc.org/github.com/NYTimes/gizmo/auth) diff --git a/server/kit/README.md b/server/kit/README.md index 5f9348655..602a655f8 100644 --- a/server/kit/README.md +++ b/server/kit/README.md @@ -1,10 +1,17 @@ -# This is an experimental package in Gizmo! +# Welcome to the 2nd generation of Gizmo servers 🚀! -* The rationale behind this package: - * A more opinionated server with fewer choices. - * go-kit is used for serving HTTP/JSON & gRPC is used for serving HTTP2/RPC - * Logs always go to stdout/stderr by default, but if running on App Engine, trace enabled Stackdriver logging will be used instead. - * Using Go's 1.8 graceful HTTP shutdown - * Services using this package are meant for deploy to GCP. +Gizmo's intentions from the beginning were to eventually join forces with the wonders of the [go-kit toolkit](https://github.com/go-kit/kit). This package is meant to embody that goal. -* If you experience any issues please create an issue and/or reach out on the #gizmo channel with what you've found. +The `kit` server is composed of multiple [kit/transport/http.Servers](https://godoc.org/github.com/go-kit/kit/transport/http#Server) that are tied together with a common HTTP mux, HTTP options and middlewares. By default all HTTP endpoints will be encoded as JSON, but developers may override each [HTTPEndpoint](https://godoc.org/github.com/NYTimes/gizmo/server/kit#HTTPEndpoint) to use whatever encoding they need. If users need to use gRPC, they can can register the same endpoints to serve both HTTP and gRPC requests on two different ports. + +This server expects to be configured via environment variables. The available variables can be found by inspecting the [Config struct](https://godoc.org/github.com/NYTimes/gizmo/server/kit#Config) within this package. If no health check or [warm up](https://cloud.google.com/appengine/docs/standard/go111/how-instances-are-managed#warmup_requests) endpoints are defined, this server will automatically register basic endpoints there to return a simple "200 OK" response. + +Since NYT uses Google Cloud, deploying this server to that environment provides additional perks: + +* If running in the [App Engine 2nd Generation runtime (Go >=1.11)](https://cloud.google.com/appengine/docs/standard/go111/), servers will: + * Automatically catch any panics and send them to Stackdriver Error reporting + * Automatically use Stackdriver logging and, if `kit.LogXXX` functions are used, logs will be trace enabled and will be combined with their parent access log in the Stackdriver logging console. + * Automatically register Stackdriver exporters for Open Census trace and monitoring. Most Google Cloud clients (like [Cloud Spanner](https://godoc.org/cloud.google.com/go/spanner)) will detect this and emit the traces. Users can also add their own trace and monitoring spans via [the Open Census clients](https://godoc.org/go.opencensus.io/trace#example-StartSpan). + + +For an example of how to build a server that utilizes this package, see the [Reading List example](https://github.com/NYTimes/gizmo/tree/master/examples/servers/reading-list#the-reading-list-example). diff --git a/server/kit/gae_log.go b/server/kit/gae_log.go index f48137753..1d02b321f 100644 --- a/server/kit/gae_log.go +++ b/server/kit/gae_log.go @@ -5,6 +5,7 @@ import ( "encoding" "encoding/json" "fmt" + "os" "reflect" "strings" @@ -15,6 +16,17 @@ import ( "google.golang.org/genproto/googleapis/api/monitoredres" ) +// project, service, version +func getGAEInfo() (string, string, string) { + return os.Getenv("GOOGLE_CLOUD_PROJECT"), + os.Getenv("GAE_SERVICE"), + os.Getenv("GAE_VERSION") +} + +func isGAE() bool { + return os.Getenv("GAE_DEPLOYMENT_ID") != "" +} + type gaeLogger struct { project string monRes *monitoredres.MonitoredResource diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index 6c6ed9c74..57ed690ad 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -1,3 +1,4 @@ +// Package kit implements an opinionated server based on go-kit primitives. package kit import ( @@ -7,7 +8,6 @@ import ( "net" "net/http" "net/http/pprof" - "os" "strings" "cloud.google.com/go/errorreporting" @@ -76,29 +76,20 @@ func NewServer(svc Service) *Server { r = opt(r) } - // check if we're running on GAE via env variables - projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") - serviceID := os.Getenv("GAE_SERVICE") - svcVersion := os.Getenv("GAE_VERSION") + ctx := context.Background() + + lg, logClose, err := NewLogger(ctx) + if err != nil { + stdlog.Fatalf("unable to start up logger: %s", err) + } var ( - err error - lg log.Logger - logClose func() error - errs *errorreporting.Client + errs *errorreporting.Client + propr propagation.HTTPFormat ) - var propr propagation.HTTPFormat - // use the version variable to determine if we're in the GAE environment - if svcVersion == "" { - lg = log.NewJSONLogger(log.NewSyncWriter(os.Stdout)) - } else { - ctx := context.Background() - lg, logClose, err = newAppEngineLogger(ctx, - projectID, serviceID, svcVersion) - if err != nil { - stdlog.Fatalf("unable to start up app engine logger: %s", err) - } - + // if in App Engine, initiate an error reporter and the stackdriver exporters + if isGAE() { + projectID, serviceID, svcVersion := getGAEInfo() errs, err = errorreporting.NewClient(ctx, projectID, errorreporting.Config{ ServiceName: serviceID, ServiceVersion: svcVersion, diff --git a/server/kit/log.go b/server/kit/log.go index 359d9bdab..05aacc25a 100644 --- a/server/kit/log.go +++ b/server/kit/log.go @@ -2,12 +2,31 @@ package kit import ( "context" + "os" "github.com/go-kit/kit/log" "github.com/go-kit/kit/transport/http" "google.golang.org/grpc/metadata" ) +// NewLogger will inspect the environment and, if running in the Google App Engine +// environment, it will return a new Stackdriver logger annotated with the current +// server's project ID, service ID and version. If not in App Engine, a normal JSON +// logger pointing to stdout will be returned. +// This function can be used for services that need to log information outside the +// context of an inbound request. +// When using the Stackdriver logger, any go-kit/log/levels will be translated to +// Stackdriver severity levels. +func NewLogger(ctx context.Context) (log.Logger, func() error, error) { + // running locally or in a non-GAE environment? use JSON + if !isGAE() { + return log.NewJSONLogger(log.NewSyncWriter(os.Stdout)), func() error { return nil }, nil + } + + projectID, serviceID, svcVersion := getGAEInfo() + return newAppEngineLogger(ctx, projectID, serviceID, svcVersion) +} + // Logger will return a kit/log.Logger that has been injected into the context by the kit // server. This logger has had request headers and metadata added as key values. // This function will only work within the scope of a request initiated by the server. diff --git a/server/kit/server.go b/server/kit/server.go index 7bd143a84..e068ed066 100644 --- a/server/kit/server.go +++ b/server/kit/server.go @@ -1,5 +1,3 @@ -// +build !appengine - package kit import ( @@ -8,9 +6,6 @@ import ( "syscall" ) -// TODO(jprobinson): built in stackdriver error reporting -// TODO(jprobinson): built in stackdriver tracing (sampling) - // Run will use environment variables to configure the server then register the given // Service and start up the server(s). // This will block until the server shuts down. diff --git a/server/kit/server_gae.go b/server/kit/server_gae.go deleted file mode 100644 index 74bfcfaea..000000000 --- a/server/kit/server_gae.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build appengine - -package kit - -import "net/http" - -// Run will not actually start a server if in the App Engine environment. -func Run(service Service) error { - http.Handle("/", NewServer(service)) - return nil -}