Merge pull request #5 from speakeasy-api/support-other-routers-and-path-hints

feat: added support for alternate middleware, the ability to provide path hints and request limits

Author: TristanSpeakEasy

.golangci.yaml

@@ -12,34 +12,53 @@ linters:
     - gochecknoglobals
     - paralleltest
     - dupl
-    - containedctx
-    # Deprecated
     - golint
     - maligned
+    # deprecated/archived
     - interfacer
     - scopelint
 issues:
+  include:
+    - EXC0002
+    - EXC0011
+    - EXC0012
+    - EXC0013
+    - EXC0014
+    - EXC0015
   exclude:
     - "returns unexported type"
     - "unlambda"
-    - "should rewrite http.NewRequestWithContext"
   exclude-rules:
     # Exclude some linters from running on tests files.
     - path: _test\.go
       linters:
-        - scopelint
         - goerr113
         - funlen
+        - godot
+        - dupl
         - gocognit
         - cyclop
+        - noctx
         - nosnakecase
+        - maintidx
     - path: _exports_test\.go
       linters:
         - testpackage
-  include:
-    - EXC0012
-    - EXC0013
-    - EXC0014
-    - EXC0015
+    - path: cmd/*
+      linters:
+        - funlen
+linters-settings:
+  tagliatelle:
+    case:
+      use-field-name: true
+      rules:
+        json: snake
+        yaml: camel
+  revive:
+    rules:
+      - name: var-naming
+        arguments: [["API"], []]
+  stylecheck:
+    checks: ["all", "-ST1000", "-ST1003"]
 run:
   go: "1.14"

capture.go

@@ -0,0 +1,307 @@
+package speakeasy
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/chromedp/cdproto/har"
+	"github.com/speakeasy-api/speakeasy-go-sdk/internal/log"
+	"github.com/speakeasy-api/speakeasy-schemas/grpc/go/registry/ingest"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+)
+
+var maxCaptureSize = 9 * 1024 * 1024
+
+var timeNow = func() time.Time {
+	return time.Now()
+}
+
+var timeSince = func(t time.Time) time.Duration {
+	return time.Since(t)
+}
+
+type handlerFunc func(http.ResponseWriter, *http.Request) error
+
+func (s *speakeasy) handleRequestResponse(w http.ResponseWriter, r *http.Request, next http.HandlerFunc, capturePathHint func(r *http.Request) string) {
+	err := s.handleRequestResponseError(w, r, func(w http.ResponseWriter, r *http.Request) error {
+		next.ServeHTTP(w, r)
+		return nil
+	}, capturePathHint)
+	if err != nil {
+		log.Logger().Error("speakeasy-sdk: unexpected error from non-error handlerFunc", zap.Error(err))
+	}
+}
+
+func (s *speakeasy) handleRequestResponseError(w http.ResponseWriter, r *http.Request, next handlerFunc, capturePathHint func(r *http.Request) string) error {
+	startTime := timeNow()
+
+	cw := NewCaptureWriter(w, maxCaptureSize)
+
+	if r.Body != nil {
+		// We need to duplicate the request body, because it should be consumed by the next handler first before we can read it
+		// (as io.Reader is a stream and can only be read once) but we are potentially storing a large request (such as a file upload)
+		// in memory, so we may need to allow the middleware to be configured to not read the body or have a max size
+		tee := io.TeeReader(r.Body, cw.GetRequestWriter())
+		r.Body = ioutil.NopCloser(tee)
+	}
+
+	ctx, c := contextWithController(r.Context())
+	r = r.WithContext(ctx)
+
+	err := next(cw.GetResponseWriter(), r)
+
+	pathHint := capturePathHint(r)
+
+	// if developer has provided a path hint use it, otherwise use the pathHint from the request
+	if c.pathHint != "" {
+		pathHint = c.pathHint
+	}
+
+	// Assuming response is done
+	go s.captureRequestResponse(cw, r, startTime, pathHint)
+
+	return err
+}
+
+func (s *speakeasy) captureRequestResponse(cw *captureWriter, r *http.Request, startTime time.Time, pathHint string) {
+	var ctx context.Context = valueOnlyContext{r.Context()}
+
+	if cw.IsReqValid() && cw.GetReqBuffer().Len() == 0 && r.Body != nil {
+		// Read the body just in case it was not read in the handler
+		if _, err := io.Copy(ioutil.Discard, r.Body); err != nil {
+			log.From(ctx).Error("speakeasy-sdk: failed to read request body", zap.Error(err))
+		}
+	}
+
+	opts := []grpc.DialOption{}
+
+	if s.secure {
+		// nolint: gosec
+		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+
+	if s.config.GRPCDialer != nil {
+		opts = append(opts, grpc.WithContextDialer(s.config.GRPCDialer()))
+	}
+
+	conn, err := grpc.DialContext(ctx, s.serverURL, opts...)
+	if err != nil {
+		log.From(ctx).Error("speakeasy-sdk: failed to create grpc connection", zap.Error(err))
+		return
+	}
+	defer conn.Close()
+
+	harData, err := json.Marshal(s.buildHarFile(ctx, cw, r, startTime))
+	if err != nil {
+		log.From(ctx).Error("speakeasy-sdk: failed to ingest request body", zap.Error(err))
+		return
+	}
+
+	ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("x-api-key", s.config.APIKey))
+
+	_, err = ingest.NewIngestServiceClient(conn).Ingest(ctx, &ingest.IngestRequest{
+		Har:      string(harData),
+		PathHint: pathHint,
+	})
+	if err != nil {
+		log.From(ctx).Error("speakeasy-sdk: failed to send ingest request", zap.Error(err))
+		return
+	}
+}
+
+func (s *speakeasy) buildHarFile(ctx context.Context, cw *captureWriter, r *http.Request, startTime time.Time) *har.HAR {
+	return &har.HAR{
+		Log: &har.Log{
+			Version: "1.2",
+			Creator: &har.Creator{
+				Name:    sdkName,
+				Version: speakeasyVersion,
+			},
+			Comment: "request capture for " + r.URL.String(),
+			Entries: []*har.Entry{
+				{
+					StartedDateTime: startTime.Format(time.RFC3339),
+					Time:            float64(timeSince(startTime).Milliseconds()),
+					Request:         s.getHarRequest(ctx, cw, r),
+					Response:        s.getHarResponse(ctx, cw, r, startTime),
+					Connection:      r.URL.Port(),
+					ServerIPAddress: r.URL.Hostname(),
+				},
+			},
+		},
+	}
+}
+
+// nolint:cyclop,funlen
+func (s *speakeasy) getHarRequest(ctx context.Context, cw *captureWriter, r *http.Request) *har.Request {
+	reqHeaders := []*har.NameValuePair{}
+	for k, v := range r.Header {
+		for _, vv := range v {
+			reqHeaders = append(reqHeaders, &har.NameValuePair{Name: k, Value: vv})
+		}
+	}
+
+	reqQueryParams := []*har.NameValuePair{}
+
+	for k, v := range r.URL.Query() {
+		for _, vv := range v {
+			reqQueryParams = append(reqQueryParams, &har.NameValuePair{Name: k, Value: vv})
+		}
+	}
+
+	reqCookies := getHarCookies(r.Cookies(), time.Time{})
+	bodyText := "--dropped--"
+	if cw.IsReqValid() {
+		bodyText = cw.GetReqBuffer().String()
+	}
+
+	hw := httptest.NewRecorder()
+
+	for k, vv := range r.Header {
+		for _, v := range vv {
+			hw.Header().Set(k, v)
+		}
+	}
+
+	b := bytes.NewBuffer([]byte{})
+	headerSize := -1
+	if err := hw.Header().Write(b); err != nil {
+		log.From(ctx).Error("speakeasy-sdk: failed to read length of request headers", zap.Error(err))
+	} else {
+		headerSize = b.Len()
+	}
+
+	var postData *har.PostData
+	if len(bodyText) > 0 {
+		reqContentType := r.Header.Get("Content-Type")
+		if reqContentType == "" {
+			reqContentType = http.DetectContentType(cw.GetReqBuffer().Bytes())
+			if reqContentType == "" {
+				reqContentType = "application/octet-stream" // default http content type
+			}
+		}
+
+		postData = &har.PostData{
+			MimeType: reqContentType,
+			Text:     bodyText,
+			Params:   []*har.Param{}, // We don't parse the body here to populate this
+		}
+	}
+
+	return &har.Request{
+		Method:      r.Method,
+		URL:         r.URL.String(),
+		Headers:     reqHeaders,
+		QueryString: reqQueryParams,
+		BodySize:    r.ContentLength,
+		PostData:    postData,
+		HTTPVersion: r.Proto,
+		Cookies:     reqCookies,
+		HeadersSize: int64(headerSize),
+	}
+}
+
+func (s *speakeasy) getHarResponse(ctx context.Context, cw *captureWriter, r *http.Request, startTime time.Time) *har.Response {
+	resHeaders := []*har.NameValuePair{}
+
+	cookieParser := http.Response{Header: http.Header{}}
+
+	for k, v := range cw.origResW.Header() {
+		for _, vv := range v {
+			if k == "Set-Cookie" {
+				cookieParser.Header.Add(k, vv)
+			}
+
+			resHeaders = append(resHeaders, &har.NameValuePair{Name: k, Value: vv})
+		}
+	}
+
+	resCookies := getHarCookies(cookieParser.Cookies(), startTime)
+
+	resContentType := cw.origResW.Header().Get("Content-Type")
+	if resContentType == "" {
+		resContentType = "application/octet-stream" // default http content type
+	}
+
+	bodyText := "--dropped--"
+	contentBodySize := 0
+	if cw.IsResValid() {
+		bodyText = cw.GetResBuffer().String()
+		contentBodySize = cw.GetResBuffer().Len()
+	}
+
+	bodySize := cw.GetResponseSize()
+	if cw.GetStatus() == http.StatusNotModified {
+		bodySize = 0
+	}
+
+	b := bytes.NewBuffer([]byte{})
+	headerSize := -1
+	if err := cw.origResW.Header().Write(b); err != nil {
+		log.From(ctx).Error("speakeasy-sdk: failed to read length of response headers", zap.Error(err))
+	} else {
+		headerSize = b.Len()
+	}
+
+	return &har.Response{
+		Status:      int64(cw.status),
+		StatusText:  http.StatusText(cw.status),
+		HTTPVersion: r.Proto,
+		Headers:     resHeaders,
+		Cookies:     resCookies,
+		Content: &har.Content{ // we are assuming we are getting the raw response here, so if we are put in the chain such that compression or encoding happens then the response text will be unreadable
+			Size:     int64(contentBodySize),
+			MimeType: resContentType,
+			Text:     bodyText,
+		},
+		RedirectURL: cw.origResW.Header().Get("Location"),
+		HeadersSize: int64(headerSize),
+		BodySize:    int64(bodySize),
+	}
+}
+
+func getHarCookies(cookies []*http.Cookie, startTime time.Time) []*har.Cookie {
+	harCookies := []*har.Cookie{}
+	for _, cookie := range cookies {
+		harCookie := &har.Cookie{
+			Name:     cookie.Name,
+			Value:    cookie.Value,
+			Path:     cookie.Path,
+			Domain:   cookie.Domain,
+			Secure:   cookie.Secure,
+			HTTPOnly: cookie.HttpOnly,
+		}
+
+		if cookie.MaxAge != 0 {
+			harCookie.Expires = startTime.Add(time.Duration(cookie.MaxAge) * time.Second).Format(time.RFC3339)
+		} else if (cookie.Expires != time.Time{}) {
+			harCookie.Expires = cookie.Expires.Format(time.RFC3339)
+		}
+
+		harCookies = append(harCookies, harCookie)
+	}
+
+	return harCookies
+}
+
+// This allows us to not be affected by context cancellation of the request that spawned our request capture while still retaining any context values.
+// nolint:containedctx
+type valueOnlyContext struct{ context.Context }
+
+// nolint
+func (valueOnlyContext) Deadline() (deadline time.Time, ok bool) { return }
+func (valueOnlyContext) Done() <-chan struct{}                   { return nil }
+func (valueOnlyContext) Err() error                              { return nil }