From 51986b2e7c183e4b9fe11318be0b232fcbc79a68 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Thu, 18 Feb 2021 17:06:40 +0800 Subject: [PATCH 01/24] =?UTF-8?q?=20=F0=9F=94=A5=20Fiber=20Client=20poc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 569 +++++++++++++++++++++++++++++++++++++++++++ client_test.go | 636 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1205 insertions(+) create mode 100644 client.go create mode 100644 client_test.go diff --git a/client.go b/client.go new file mode 100644 index 00000000..7bacade9 --- /dev/null +++ b/client.go @@ -0,0 +1,569 @@ +package fiber + +import ( + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/gofiber/fiber/v2/internal/encoding/json" + "github.com/valyala/fasthttp" +) + +// Request represents HTTP request. +// +// It is forbidden copying Request instances. Create new instances +// and use CopyTo instead. +// +// Request instance MUST NOT be used from concurrently running goroutines. +type Request = fasthttp.Request + +// Response represents HTTP response. +// +// It is forbidden copying Response instances. Create new instances +// and use CopyTo instead. +// +// Response instance MUST NOT be used from concurrently running goroutines. +type Response = fasthttp.Response + +// Args represents query arguments. +// +// It is forbidden copying Args instances. Create new instances instead +// and use CopyTo(). +// +// Args instance MUST NOT be used from concurrently running goroutines. +type Args = fasthttp.Args + +var defaultClient Client + +// Client implements http client. +// +// It is safe calling Client methods from concurrently running goroutines. +type Client struct { + UserAgent string + NoDefaultUserAgentHeader bool +} + +// Get returns a agent with http method GET. +func Get(url string) *Agent { return defaultClient.Get(url) } + +// Get returns a agent with http method GET. +func (c *Client) Get(url string) *Agent { + return c.createAgent(MethodGet, url) +} + +// Post sends POST request to the given url. +func Post(url string) *Agent { return defaultClient.Post(url) } + +// Post sends POST request to the given url. +func (c *Client) Post(url string) *Agent { + return c.createAgent(MethodPost, url) +} + +func (c *Client) createAgent(method, url string) *Agent { + a := AcquireAgent() + a.req.Header.SetMethod(method) + a.req.SetRequestURI(url) + + a.Name = c.UserAgent + a.NoDefaultUserAgentHeader = c.NoDefaultUserAgentHeader + + if err := a.Parse(); err != nil { + a.errs = append(a.errs, err) + } + + return a +} + +// Agent is an object storing all request data for client. +type Agent struct { + *fasthttp.HostClient + req *Request + customReq *Request + args *Args + timeout time.Duration + errs []error + debugWriter io.Writer + maxRedirectsCount int + Name string + NoDefaultUserAgentHeader bool + reuse bool + parsed bool +} + +var ErrorInvalidURI = fasthttp.ErrorInvalidURI + +// Parse initializes URI and HostClient. +func (a *Agent) Parse() error { + if a.parsed { + return nil + } + a.parsed = true + + req := a.req + if a.customReq != nil { + req = a.customReq + } + + uri := req.URI() + if uri == nil { + return ErrorInvalidURI + } + + isTLS := false + scheme := uri.Scheme() + if bytes.Equal(scheme, strHTTPS) { + isTLS = true + } else if !bytes.Equal(scheme, strHTTP) { + return fmt.Errorf("unsupported protocol %q. http and https are supported", scheme) + } + + name := a.Name + if name == "" && !a.NoDefaultUserAgentHeader { + name = defaultUserAgent + } + + a.HostClient = &fasthttp.HostClient{ + Addr: addMissingPort(string(uri.Host()), isTLS), + Name: name, + NoDefaultUserAgentHeader: a.NoDefaultUserAgentHeader, + IsTLS: isTLS, + } + + return nil +} + +func addMissingPort(addr string, isTLS bool) string { + n := strings.Index(addr, ":") + if n >= 0 { + return addr + } + port := 80 + if isTLS { + port = 443 + } + return net.JoinHostPort(addr, strconv.Itoa(port)) +} + +// Set sets the given 'key: value' header. +// +// Use Add for setting multiple header values under the same key. +func (a *Agent) Set(k, v string) *Agent { + a.req.Header.Set(k, v) + + return a +} + +// Add adds the given 'key: value' header. +// +// Multiple headers with the same key may be added with this function. +// Use Set for setting a single header for the given key. +func (a *Agent) Add(k, v string) *Agent { + a.req.Header.Add(k, v) + + return a +} + +// Host sets host for the uri. +func (a *Agent) Host(host string) *Agent { + a.req.URI().SetHost(host) + + return a +} + +// ConnectionClose sets 'Connection: close' header. +func (a *Agent) ConnectionClose() *Agent { + a.req.Header.SetConnectionClose() + + return a +} + +// UserAgent sets User-Agent header value. +func (a *Agent) UserAgent(userAgent string) *Agent { + a.req.Header.SetUserAgent(userAgent) + + return a +} + +// Debug mode enables logging request and response detail +func (a *Agent) Debug(w ...io.Writer) *Agent { + a.debugWriter = os.Stdout + if len(w) > 0 { + a.debugWriter = w[0] + } + + return a +} + +// Cookie sets one 'key: value' cookie. +func (a *Agent) Cookie(key, value string) *Agent { + a.req.Header.SetCookie(key, value) + + return a +} + +// Cookies sets multiple 'key: value' cookies. +func (a *Agent) Cookies(kv ...string) *Agent { + for i := 1; i < len(kv); i += 2 { + a.req.Header.SetCookie(kv[i-1], kv[i]) + } + + return a +} + +// Timeout sets request timeout duration. +func (a *Agent) Timeout(timeout time.Duration) *Agent { + a.timeout = timeout + + return a +} + +// Json sends a json request. +func (a *Agent) Json(v interface{}) *Agent { + a.req.Header.SetContentType(MIMEApplicationJSON) + + if body, err := json.Marshal(v); err != nil { + a.errs = append(a.errs, err) + } else { + a.req.SetBody(body) + } + + return a +} + +// Form sends request with body if args is non-nil. +// +// Note that this will force http method to post. +func (a *Agent) Form(args *Args) *Agent { + a.req.Header.SetContentType(MIMEApplicationForm) + + if args != nil { + if _, err := args.WriteTo(a.req.BodyWriter()); err != nil { + a.errs = append(a.errs, err) + } + } + + return a +} + +// QueryString sets URI query string. +func (a *Agent) QueryString(queryString string) *Agent { + a.req.URI().SetQueryString(queryString) + + return a +} + +// BodyStream sets request body stream and, optionally body size. +// +// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes +// before returning io.EOF. +// +// If bodySize < 0, then bodyStream is read until io.EOF. +// +// bodyStream.Close() is called after finishing reading all body data +// if it implements io.Closer. +// +// Note that GET and HEAD requests cannot have body. +func (a *Agent) BodyStream(bodyStream io.Reader, bodySize int) *Agent { + a.req.SetBodyStream(bodyStream, bodySize) + + return a +} + +// Reuse indicates the createAgent can be used again after one request. +func (a *Agent) Reuse() *Agent { + a.reuse = true + + return a +} + +// InsecureSkipVerify controls whether the createAgent verifies the server's +// certificate chain and host name. +func (a *Agent) InsecureSkipVerify() *Agent { + if a.HostClient.TLSConfig == nil { + a.HostClient.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } else { + a.HostClient.TLSConfig.InsecureSkipVerify = true + } + + return a +} + +// TLSConfig sets tls config. +func (a *Agent) TLSConfig(config *tls.Config) *Agent { + a.HostClient.TLSConfig = config + + return a +} + +// Request sets custom request for createAgent. +func (a *Agent) Request(req *Request) *Agent { + a.customReq = req + + return a +} + +// Referer sets Referer header value. +func (a *Agent) Referer(referer string) *Agent { + a.req.Header.SetReferer(referer) + + return a +} + +// ContentType sets Content-Type header value. +func (a *Agent) ContentType(contentType string) *Agent { + a.req.Header.SetContentType(contentType) + + return a +} + +// MaxRedirectsCount sets max redirect count for GET and HEAD. +func (a *Agent) MaxRedirectsCount(count int) *Agent { + a.maxRedirectsCount = count + + return a +} + +// Bytes returns the status code, bytes body and errors of url. +func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []error) { + defer a.release() + + if errs = append(errs, a.errs...); len(errs) > 0 { + return + } + + req := a.req + if a.customReq != nil { + req = a.customReq + } + + var ( + resp *Response + releaseResp bool + ) + if len(customResp) > 0 { + resp = customResp[0] + } else { + resp = AcquireResponse() + releaseResp = true + } + defer func() { + if a.debugWriter != nil { + printDebugInfo(req, resp, a.debugWriter) + } + + if len(errs) == 0 { + code = resp.StatusCode() + } + + if releaseResp { + body = append(body, resp.Body()...) + ReleaseResponse(resp) + } else { + body = resp.Body() + } + }() + + if a.timeout > 0 { + if err := a.HostClient.DoTimeout(req, resp, a.timeout); err != nil { + errs = append(errs, err) + return + } + } + + if a.maxRedirectsCount > 0 && (string(req.Header.Method()) == MethodGet || string(req.Header.Method()) == MethodHead) { + if err := a.HostClient.DoRedirects(req, resp, a.maxRedirectsCount); err != nil { + errs = append(errs, err) + return + } + } + + if err := a.HostClient.Do(req, resp); err != nil { + errs = append(errs, err) + } + + return +} + +func printDebugInfo(req *Request, resp *Response, w io.Writer) { + msg := fmt.Sprintf("Connected to %s(%s)\r\n\r\n", req.URI().Host(), resp.RemoteAddr()) + _, _ = w.Write(getBytes(msg)) + _, _ = req.WriteTo(w) + _, _ = resp.WriteTo(w) +} + +// String returns the status code, string body and errors of url. +func (a *Agent) String(resp ...*Response) (int, string, []error) { + code, body, errs := a.Bytes(resp...) + + return code, getString(body), errs +} + +// Struct returns the status code, bytes body and errors of url. +// And bytes body will be unmarshalled to given v. +func (a *Agent) Struct(v interface{}, resp ...*Response) (code int, body []byte, errs []error) { + code, body, errs = a.Bytes(resp...) + + if err := json.Unmarshal(body, v); err != nil { + errs = append(errs, err) + } + + return +} + +func (a *Agent) release() { + if !a.reuse { + ReleaseAgent(a) + } else { + a.errs = a.errs[:0] + } +} + +func (a *Agent) reset() { + a.HostClient = nil + a.req.Reset() + a.customReq = nil + a.timeout = 0 + a.args = nil + a.errs = a.errs[:0] + a.debugWriter = nil + a.reuse = false + a.parsed = false + a.maxRedirectsCount = 0 + a.Name = "" + a.NoDefaultUserAgentHeader = false +} + +var ( + clientPool sync.Pool + agentPool sync.Pool + requestPool sync.Pool + responsePool sync.Pool + argsPool sync.Pool +) + +// AcquireAgent returns an empty Agent instance from createAgent pool. +// +// The returned Agent instance may be passed to ReleaseAgent when it is +// no longer needed. This allows Agent recycling, reduces GC pressure +// and usually improves performance. +func AcquireAgent() *Agent { + v := agentPool.Get() + if v == nil { + return &Agent{req: fasthttp.AcquireRequest()} + } + return v.(*Agent) +} + +// ReleaseAgent returns a acquired via AcquireAgent to createAgent pool. +// +// It is forbidden accessing req and/or its' members after returning +// it to createAgent pool. +func ReleaseAgent(a *Agent) { + a.reset() + agentPool.Put(a) +} + +// AcquireClient returns an empty Client instance from client pool. +// +// The returned Client instance may be passed to ReleaseClient when it is +// no longer needed. This allows Client recycling, reduces GC pressure +// and usually improves performance. +func AcquireClient() *Client { + v := clientPool.Get() + if v == nil { + return &Client{} + } + return v.(*Client) +} + +// ReleaseClient returns c acquired via AcquireClient to client pool. +// +// It is forbidden accessing req and/or its' members after returning +// it to client pool. +func ReleaseClient(c *Client) { + c.UserAgent = "" + c.NoDefaultUserAgentHeader = false + + clientPool.Put(c) +} + +// AcquireRequest returns an empty Request instance from request pool. +// +// The returned Request instance may be passed to ReleaseRequest when it is +// no longer needed. This allows Request recycling, reduces GC pressure +// and usually improves performance. +func AcquireRequest() *Request { + v := requestPool.Get() + if v == nil { + return &Request{} + } + return v.(*Request) +} + +// ReleaseRequest returns req acquired via AcquireRequest to request pool. +// +// It is forbidden accessing req and/or its' members after returning +// it to request pool. +func ReleaseRequest(req *Request) { + req.Reset() + requestPool.Put(req) +} + +// AcquireResponse returns an empty Response instance from response pool. +// +// The returned Response instance may be passed to ReleaseResponse when it is +// no longer needed. This allows Response recycling, reduces GC pressure +// and usually improves performance. +// Copy from fasthttp +func AcquireResponse() *Response { + v := responsePool.Get() + if v == nil { + return &Response{} + } + return v.(*Response) +} + +// ReleaseResponse return resp acquired via AcquireResponse to response pool. +// +// It is forbidden accessing resp and/or its' members after returning +// it to response pool. +// Copy from fasthttp +func ReleaseResponse(resp *Response) { + resp.Reset() + responsePool.Put(resp) +} + +// AcquireArgs returns an empty Args object from the pool. +// +// The returned Args may be returned to the pool with ReleaseArgs +// when no longer needed. This allows reducing GC load. +// Copy from fasthttp +func AcquireArgs() *Args { + v := argsPool.Get() + if v == nil { + return &Args{} + } + return v.(*Args) +} + +// ReleaseArgs returns the object acquired via AcquireArgs to the pool. +// +// String not access the released Args object, otherwise data races may occur. +// Copy from fasthttp +func ReleaseArgs(a *Args) { + a.Reset() + argsPool.Put(a) +} + +var ( + strHTTP = []byte("http") + strHTTPS = []byte("https") + defaultUserAgent = "fiber" +) diff --git a/client_test.go b/client_test.go new file mode 100644 index 00000000..2ff99d39 --- /dev/null +++ b/client_test.go @@ -0,0 +1,636 @@ +package fiber + +import ( + "bytes" + "crypto/tls" + "net" + "strings" + "testing" + "time" + + "github.com/gofiber/fiber/v2/utils" + "github.com/valyala/fasthttp/fasthttputil" +) + +func Test_Client_Invalid_URL(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString(c.Hostname()) + }) + + go app.Listener(ln) //nolint:errcheck + + a := Get("http://example.com\r\n\r\nGET /\r\n\r\n") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "missing required Host header in request", errs[0].Error()) +} + +func Test_Client_Unsupported_Protocol(t *testing.T) { + t.Parallel() + + a := Get("ftp://example.com") + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, `unsupported protocol "ftp". http and https are supported`, + errs[0].Error()) +} + +func Test_Client_Get(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString(c.Hostname()) + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "example.com", body) + utils.AssertEqual(t, 0, len(errs)) + } +} + +func Test_Client_Post(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Post("/", func(c *Ctx) error { + return c.SendString(c.Hostname()) + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + args := AcquireArgs() + + args.Set("foo", "bar") + + a := Post("http://example.com"). + Form(args) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "example.com", body) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseArgs(args) + } +} + +func Test_Client_UserAgent(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.Send(c.Request().Header.UserAgent()) + }) + + go app.Listener(ln) //nolint:errcheck + + t.Run("default", func(t *testing.T) { + for i := 0; i < 5; i++ { + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, defaultUserAgent, body) + utils.AssertEqual(t, 0, len(errs)) + } + }) + + t.Run("custom", func(t *testing.T) { + for i := 0; i < 5; i++ { + c := AcquireClient() + c.UserAgent = "ua" + + a := c.Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "ua", body) + utils.AssertEqual(t, 0, len(errs)) + ReleaseClient(c) + } + }) +} + +func Test_Client_Agent_Specific_Host(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString(c.Hostname()) + }) + + go app.Listener(ln) //nolint:errcheck + + a := Get("http://1.1.1.1:8080"). + Host("example.com") + + utils.AssertEqual(t, "1.1.1.1:8080", a.HostClient.Addr) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "example.com", body) + utils.AssertEqual(t, 0, len(errs)) +} + +func Test_Client_Agent_Headers(t *testing.T) { + handler := func(c *Ctx) error { + c.Request().Header.VisitAll(func(key, value []byte) { + if k := string(key); k == "K1" || k == "K2" { + _, _ = c.Write(key) + _, _ = c.Write(value) + } + }) + return nil + } + + wrapAgent := func(a *Agent) { + a.Set("k1", "v1"). + Add("k1", "v11"). + Set("k2", "v2") + } + + testAgent(t, handler, wrapAgent, "K1v1K1v11K2v2") +} + +func Test_Client_Agent_UserAgent(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.UserAgent()) + } + + wrapAgent := func(a *Agent) { + a.UserAgent("ua") + } + + testAgent(t, handler, wrapAgent, "ua") +} + +func Test_Client_Agent_Connection_Close(t *testing.T) { + handler := func(c *Ctx) error { + if c.Request().Header.ConnectionClose() { + return c.SendString("close") + } + return c.SendString("not close") + } + + wrapAgent := func(a *Agent) { + a.ConnectionClose() + } + + testAgent(t, handler, wrapAgent, "close") +} + +func Test_Client_Agent_Referer(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.Referer()) + } + + wrapAgent := func(a *Agent) { + a.Referer("http://referer.com") + } + + testAgent(t, handler, wrapAgent, "http://referer.com") +} + +func Test_Client_Agent_QueryString(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().URI().QueryString()) + } + + wrapAgent := func(a *Agent) { + a.QueryString("foo=bar&bar=baz") + } + + testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") +} + +func Test_Client_Agent_Cookie(t *testing.T) { + handler := func(c *Ctx) error { + return c.SendString( + c.Cookies("k1") + c.Cookies("k2") + c.Cookies("k3") + c.Cookies("k4")) + } + + wrapAgent := func(a *Agent) { + a.Cookie("k1", "v1"). + Cookie("k2", "v2"). + Cookies("k3", "v3", "k4", "v4") + } + + testAgent(t, handler, wrapAgent, "v1v2v3v4") +} + +func Test_Client_Agent_ContentType(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.ContentType()) + } + + wrapAgent := func(a *Agent) { + a.ContentType("custom-type") + } + + testAgent(t, handler, wrapAgent, "custom-type") +} + +func Test_Client_Agent_BodyStream(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Body()) + } + + wrapAgent := func(a *Agent) { + a.BodyStream(strings.NewReader("body stream"), -1) + } + + testAgent(t, handler, wrapAgent, "body stream") +} + +func Test_Client_Agent_Form(t *testing.T) { + handler := func(c *Ctx) error { + utils.AssertEqual(t, MIMEApplicationForm, string(c.Request().Header.ContentType())) + + return c.Send(c.Request().Body()) + } + + args := AcquireArgs() + + args.Set("a", "b") + + wrapAgent := func(a *Agent) { + a.Form(args) + } + + testAgent(t, handler, wrapAgent, "a=b") + + ReleaseArgs(args) +} + +type jsonData struct { + F string `json:"f"` +} + +func Test_Client_Agent_Json(t *testing.T) { + handler := func(c *Ctx) error { + utils.AssertEqual(t, MIMEApplicationJSON, string(c.Request().Header.ContentType())) + + return c.Send(c.Request().Body()) + } + + wrapAgent := func(a *Agent) { + a.Json(jsonData{F: "f"}) + } + + testAgent(t, handler, wrapAgent, `{"f":"f"}`) +} + +func Test_Client_Agent_Json_Error(t *testing.T) { + a := Get("http://example.com"). + Json(complex(1, 1)) + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "json: unsupported type: complex128", errs[0].Error()) +} + +func Test_Client_Debug(t *testing.T) { + handler := func(c *Ctx) error { + return c.SendString("debug") + } + + var output bytes.Buffer + + wrapAgent := func(a *Agent) { + a.Debug(&output) + } + + testAgent(t, handler, wrapAgent, "debug", 1) + + str := output.String() + + utils.AssertEqual(t, true, strings.Contains(str, "Connected to example.com(pipe)")) + utils.AssertEqual(t, true, strings.Contains(str, "GET / HTTP/1.1")) + utils.AssertEqual(t, true, strings.Contains(str, "User-Agent: fiber")) + utils.AssertEqual(t, true, strings.Contains(str, "Host: example.com\r\n\r\n")) + utils.AssertEqual(t, true, strings.Contains(str, "HTTP/1.1 200 OK")) + utils.AssertEqual(t, true, strings.Contains(str, "Content-Type: text/plain; charset=utf-8\r\nContent-Length: 5\r\n\r\ndebug")) +} + +func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), excepted string, count ...int) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", handler) + + go app.Listener(ln) //nolint:errcheck + + c := 1 + if len(count) > 0 { + c = count[0] + } + + for i := 0; i < c; i++ { + a := Get("http://example.com") + + wrapAgent(a) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, excepted, body) + utils.AssertEqual(t, 0, len(errs)) + } +} + +func Test_Client_Agent_Timeout(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + time.Sleep(time.Millisecond * 200) + return c.SendString("timeout") + }) + + go app.Listener(ln) //nolint:errcheck + + a := Get("http://example.com"). + Timeout(time.Millisecond * 100) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "timeout", errs[0].Error()) +} + +func Test_Client_Agent_MaxRedirectsCount(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + if c.Request().URI().QueryArgs().Has("foo") { + return c.Redirect("/foo") + } + return c.Redirect("/") + }) + app.Get("/foo", func(c *Ctx) error { + return c.SendString("redirect") + }) + + go app.Listener(ln) //nolint:errcheck + + t.Run("success", func(t *testing.T) { + a := Get("http://example.com?foo"). + MaxRedirectsCount(1) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, 200, code) + utils.AssertEqual(t, "redirect", body) + utils.AssertEqual(t, 0, len(errs)) + }) + + t.Run("error", func(t *testing.T) { + a := Get("http://example.com"). + MaxRedirectsCount(1) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "too many redirects detected when doing the request", errs[0].Error()) + }) +} + +func Test_Client_Agent_Custom(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("custom") + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + a := AcquireAgent() + req := AcquireRequest() + resp := AcquireResponse() + + req.Header.SetMethod(MethodGet) + req.SetRequestURI("http://example.com") + a.Request(req) + + utils.AssertEqual(t, nil, a.Parse()) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String(resp) + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "custom", body) + utils.AssertEqual(t, "custom", string(resp.Body())) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseRequest(req) + ReleaseResponse(resp) + } +} + +func Test_Client_Agent_Reuse(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("reuse") + }) + + go app.Listener(ln) //nolint:errcheck + + a := Get("http://example.com"). + Reuse() + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "reuse", body) + utils.AssertEqual(t, 0, len(errs)) + + code, body, errs = a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "reuse", body) + utils.AssertEqual(t, 0, len(errs)) +} + +func Test_Client_Agent_Parse(t *testing.T) { + t.Parallel() + + a := Get("https://example.com:10443") + + utils.AssertEqual(t, nil, a.Parse()) +} + +func Test_Client_Agent_TLS(t *testing.T) { + t.Parallel() + + // Create tls certificate + cer, err := tls.LoadX509KeyPair("./.github/testdata/ssl.pem", "./.github/testdata/ssl.key") + utils.AssertEqual(t, nil, err) + + config := &tls.Config{ + Certificates: []tls.Certificate{cer}, + } + + ln, err := net.Listen(NetworkTCP4, "127.0.0.1:0") + utils.AssertEqual(t, nil, err) + + ln = tls.NewListener(ln, config) + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("tls") + }) + + go app.Listener(ln) //nolint:errcheck + + code, body, errs := Get("https://" + ln.Addr().String()). + InsecureSkipVerify(). + TLSConfig(config). + InsecureSkipVerify(). + String() + + utils.AssertEqual(t, 0, len(errs)) + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "tls", body) +} + +type data struct { + Success bool `json:"success"` +} + +func Test_Client_Agent_Struct(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.JSON(data{true}) + }) + + app.Get("/error", func(c *Ctx) error { + return c.SendString(`{"success"`) + }) + + go app.Listener(ln) //nolint:errcheck + + t.Run("success", func(t *testing.T) { + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + var d data + + code, body, errs := a.Struct(&d) + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, `{"success":true}`, string(body)) + utils.AssertEqual(t, 0, len(errs)) + utils.AssertEqual(t, true, d.Success) + }) + + t.Run("error", func(t *testing.T) { + a := Get("http://example.com/error") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + var d data + + code, body, errs := a.Struct(&d) + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, `{"success"`, string(body)) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "json: unexpected end of JSON input after object field key: ", errs[0].Error()) + }) +} + +func Test_AddMissingPort_TLS(t *testing.T) { + addr := addMissingPort("example.com", true) + utils.AssertEqual(t, "example.com:443", addr) +} From e93306ca61226f1df7ac5f364627cf12268b2eb0 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Thu, 18 Feb 2021 17:26:13 +0800 Subject: [PATCH 02/24] =?UTF-8?q?=20=F0=9F=91=B7=20Add=20BodyString?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 7 +++++++ client_test.go | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/client.go b/client.go index 7bacade9..069c220a 100644 --- a/client.go +++ b/client.go @@ -259,6 +259,13 @@ func (a *Agent) QueryString(queryString string) *Agent { return a } +// BodyString sets request body. +func (a *Agent) BodyString(bodyString string) *Agent { + a.req.SetBodyString(bodyString) + + return a +} + // BodyStream sets request body stream and, optionally body size. // // If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes diff --git a/client_test.go b/client_test.go index 2ff99d39..16c0c850 100644 --- a/client_test.go +++ b/client_test.go @@ -252,6 +252,18 @@ func Test_Client_Agent_QueryString(t *testing.T) { testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") } +func Test_Client_Agent_BodyString(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Body()) + } + + wrapAgent := func(a *Agent) { + a.BodyString("foo=bar&bar=baz") + } + + testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") +} + func Test_Client_Agent_Cookie(t *testing.T) { handler := func(c *Ctx) error { return c.SendString( From 6cc4a5ec047bff647e6560ace04feacb1e0a1dba Mon Sep 17 00:00:00 2001 From: Kiyon Date: Fri, 19 Feb 2021 09:01:54 +0800 Subject: [PATCH 03/24] =?UTF-8?q?=F0=9F=8D=80=20Cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 144 +++++++------- client_test.go | 506 ++++++++++++++++++++++++------------------------- 2 files changed, 333 insertions(+), 317 deletions(-) diff --git a/client.go b/client.go index 069c220a..c8e78d17 100644 --- a/client.go +++ b/client.go @@ -151,6 +151,8 @@ func addMissingPort(addr string, isTLS bool) string { return net.JoinHostPort(addr, strconv.Itoa(port)) } +/************************** Header Setting **************************/ + // Set sets the given 'key: value' header. // // Use Add for setting multiple header values under the same key. @@ -170,13 +172,6 @@ func (a *Agent) Add(k, v string) *Agent { return a } -// Host sets host for the uri. -func (a *Agent) Host(host string) *Agent { - a.req.URI().SetHost(host) - - return a -} - // ConnectionClose sets 'Connection: close' header. func (a *Agent) ConnectionClose() *Agent { a.req.Header.SetConnectionClose() @@ -191,16 +186,6 @@ func (a *Agent) UserAgent(userAgent string) *Agent { return a } -// Debug mode enables logging request and response detail -func (a *Agent) Debug(w ...io.Writer) *Agent { - a.debugWriter = os.Stdout - if len(w) > 0 { - a.debugWriter = w[0] - } - - return a -} - // Cookie sets one 'key: value' cookie. func (a *Agent) Cookie(key, value string) *Agent { a.req.Header.SetCookie(key, value) @@ -217,9 +202,69 @@ func (a *Agent) Cookies(kv ...string) *Agent { return a } -// Timeout sets request timeout duration. -func (a *Agent) Timeout(timeout time.Duration) *Agent { - a.timeout = timeout +// Referer sets Referer header value. +func (a *Agent) Referer(referer string) *Agent { + a.req.Header.SetReferer(referer) + + return a +} + +// ContentType sets Content-Type header value. +func (a *Agent) ContentType(contentType string) *Agent { + a.req.Header.SetContentType(contentType) + + return a +} + +/************************** End Header Setting **************************/ + +/************************** URI Setting **************************/ + +// Host sets host for the uri. +func (a *Agent) Host(host string) *Agent { + a.req.URI().SetHost(host) + + return a +} + +// QueryString sets URI query string. +func (a *Agent) QueryString(queryString string) *Agent { + a.req.URI().SetQueryString(queryString) + + return a +} + +/************************** End URI Setting **************************/ + +/************************** Request Setting **************************/ + +// BodyString sets request body. +func (a *Agent) BodyString(bodyString string) *Agent { + a.req.SetBodyString(bodyString) + + return a +} + +// BodyStream sets request body stream and, optionally body size. +// +// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes +// before returning io.EOF. +// +// If bodySize < 0, then bodyStream is read until io.EOF. +// +// bodyStream.Close() is called after finishing reading all body data +// if it implements io.Closer. +// +// Note that GET and HEAD requests cannot have body. +func (a *Agent) BodyStream(bodyStream io.Reader, bodySize int) *Agent { + a.req.SetBodyStream(bodyStream, bodySize) + + return a +} + +// Request sets custom request for createAgent. +func (a *Agent) Request(req *Request) *Agent { + a.customReq = req return a } @@ -252,33 +297,23 @@ func (a *Agent) Form(args *Args) *Agent { return a } -// QueryString sets URI query string. -func (a *Agent) QueryString(queryString string) *Agent { - a.req.URI().SetQueryString(queryString) +/************************** End Request Setting **************************/ + +/************************** Agent Setting **************************/ + +// Debug mode enables logging request and response detail +func (a *Agent) Debug(w ...io.Writer) *Agent { + a.debugWriter = os.Stdout + if len(w) > 0 { + a.debugWriter = w[0] + } return a } -// BodyString sets request body. -func (a *Agent) BodyString(bodyString string) *Agent { - a.req.SetBodyString(bodyString) - - return a -} - -// BodyStream sets request body stream and, optionally body size. -// -// If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes -// before returning io.EOF. -// -// If bodySize < 0, then bodyStream is read until io.EOF. -// -// bodyStream.Close() is called after finishing reading all body data -// if it implements io.Closer. -// -// Note that GET and HEAD requests cannot have body. -func (a *Agent) BodyStream(bodyStream io.Reader, bodySize int) *Agent { - a.req.SetBodyStream(bodyStream, bodySize) +// Timeout sets request timeout duration. +func (a *Agent) Timeout(timeout time.Duration) *Agent { + a.timeout = timeout return a } @@ -309,27 +344,6 @@ func (a *Agent) TLSConfig(config *tls.Config) *Agent { return a } -// Request sets custom request for createAgent. -func (a *Agent) Request(req *Request) *Agent { - a.customReq = req - - return a -} - -// Referer sets Referer header value. -func (a *Agent) Referer(referer string) *Agent { - a.req.Header.SetReferer(referer) - - return a -} - -// ContentType sets Content-Type header value. -func (a *Agent) ContentType(contentType string) *Agent { - a.req.Header.SetContentType(contentType) - - return a -} - // MaxRedirectsCount sets max redirect count for GET and HEAD. func (a *Agent) MaxRedirectsCount(count int) *Agent { a.maxRedirectsCount = count @@ -337,6 +351,8 @@ func (a *Agent) MaxRedirectsCount(count int) *Agent { return a } +/************************** End Agent Setting **************************/ + // Bytes returns the status code, bytes body and errors of url. func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []error) { defer a.release() diff --git a/client_test.go b/client_test.go index 16c0c850..1e952bcf 100644 --- a/client_test.go +++ b/client_test.go @@ -154,6 +154,92 @@ func Test_Client_UserAgent(t *testing.T) { }) } +func Test_Client_Agent_Headers(t *testing.T) { + handler := func(c *Ctx) error { + c.Request().Header.VisitAll(func(key, value []byte) { + if k := string(key); k == "K1" || k == "K2" { + _, _ = c.Write(key) + _, _ = c.Write(value) + } + }) + return nil + } + + wrapAgent := func(a *Agent) { + a.Set("k1", "v1"). + Add("k1", "v11"). + Set("k2", "v2") + } + + testAgent(t, handler, wrapAgent, "K1v1K1v11K2v2") +} + +func Test_Client_Agent_Connection_Close(t *testing.T) { + handler := func(c *Ctx) error { + if c.Request().Header.ConnectionClose() { + return c.SendString("close") + } + return c.SendString("not close") + } + + wrapAgent := func(a *Agent) { + a.ConnectionClose() + } + + testAgent(t, handler, wrapAgent, "close") +} + +func Test_Client_Agent_UserAgent(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.UserAgent()) + } + + wrapAgent := func(a *Agent) { + a.UserAgent("ua") + } + + testAgent(t, handler, wrapAgent, "ua") +} + +func Test_Client_Agent_Cookie(t *testing.T) { + handler := func(c *Ctx) error { + return c.SendString( + c.Cookies("k1") + c.Cookies("k2") + c.Cookies("k3") + c.Cookies("k4")) + } + + wrapAgent := func(a *Agent) { + a.Cookie("k1", "v1"). + Cookie("k2", "v2"). + Cookies("k3", "v3", "k4", "v4") + } + + testAgent(t, handler, wrapAgent, "v1v2v3v4") +} + +func Test_Client_Agent_Referer(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.Referer()) + } + + wrapAgent := func(a *Agent) { + a.Referer("http://referer.com") + } + + testAgent(t, handler, wrapAgent, "http://referer.com") +} + +func Test_Client_Agent_ContentType(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Header.ContentType()) + } + + wrapAgent := func(a *Agent) { + a.ContentType("custom-type") + } + + testAgent(t, handler, wrapAgent, "custom-type") +} + func Test_Client_Agent_Specific_Host(t *testing.T) { t.Parallel() @@ -181,65 +267,6 @@ func Test_Client_Agent_Specific_Host(t *testing.T) { utils.AssertEqual(t, 0, len(errs)) } -func Test_Client_Agent_Headers(t *testing.T) { - handler := func(c *Ctx) error { - c.Request().Header.VisitAll(func(key, value []byte) { - if k := string(key); k == "K1" || k == "K2" { - _, _ = c.Write(key) - _, _ = c.Write(value) - } - }) - return nil - } - - wrapAgent := func(a *Agent) { - a.Set("k1", "v1"). - Add("k1", "v11"). - Set("k2", "v2") - } - - testAgent(t, handler, wrapAgent, "K1v1K1v11K2v2") -} - -func Test_Client_Agent_UserAgent(t *testing.T) { - handler := func(c *Ctx) error { - return c.Send(c.Request().Header.UserAgent()) - } - - wrapAgent := func(a *Agent) { - a.UserAgent("ua") - } - - testAgent(t, handler, wrapAgent, "ua") -} - -func Test_Client_Agent_Connection_Close(t *testing.T) { - handler := func(c *Ctx) error { - if c.Request().Header.ConnectionClose() { - return c.SendString("close") - } - return c.SendString("not close") - } - - wrapAgent := func(a *Agent) { - a.ConnectionClose() - } - - testAgent(t, handler, wrapAgent, "close") -} - -func Test_Client_Agent_Referer(t *testing.T) { - handler := func(c *Ctx) error { - return c.Send(c.Request().Header.Referer()) - } - - wrapAgent := func(a *Agent) { - a.Referer("http://referer.com") - } - - testAgent(t, handler, wrapAgent, "http://referer.com") -} - func Test_Client_Agent_QueryString(t *testing.T) { handler := func(c *Ctx) error { return c.Send(c.Request().URI().QueryString()) @@ -264,33 +291,6 @@ func Test_Client_Agent_BodyString(t *testing.T) { testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") } -func Test_Client_Agent_Cookie(t *testing.T) { - handler := func(c *Ctx) error { - return c.SendString( - c.Cookies("k1") + c.Cookies("k2") + c.Cookies("k3") + c.Cookies("k4")) - } - - wrapAgent := func(a *Agent) { - a.Cookie("k1", "v1"). - Cookie("k2", "v2"). - Cookies("k3", "v3", "k4", "v4") - } - - testAgent(t, handler, wrapAgent, "v1v2v3v4") -} - -func Test_Client_Agent_ContentType(t *testing.T) { - handler := func(c *Ctx) error { - return c.Send(c.Request().Header.ContentType()) - } - - wrapAgent := func(a *Agent) { - a.ContentType("custom-type") - } - - testAgent(t, handler, wrapAgent, "custom-type") -} - func Test_Client_Agent_BodyStream(t *testing.T) { handler := func(c *Ctx) error { return c.Send(c.Request().Body()) @@ -303,28 +303,42 @@ func Test_Client_Agent_BodyStream(t *testing.T) { testAgent(t, handler, wrapAgent, "body stream") } -func Test_Client_Agent_Form(t *testing.T) { - handler := func(c *Ctx) error { - utils.AssertEqual(t, MIMEApplicationForm, string(c.Request().Header.ContentType())) +func Test_Client_Agent_Custom_Request_And_Response(t *testing.T) { + t.Parallel() - return c.Send(c.Request().Body()) + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("custom") + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + a := AcquireAgent() + req := AcquireRequest() + resp := AcquireResponse() + + req.Header.SetMethod(MethodGet) + req.SetRequestURI("http://example.com") + a.Request(req) + + utils.AssertEqual(t, nil, a.Parse()) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String(resp) + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "custom", body) + utils.AssertEqual(t, "custom", string(resp.Body())) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseRequest(req) + ReleaseResponse(resp) } - - args := AcquireArgs() - - args.Set("a", "b") - - wrapAgent := func(a *Agent) { - a.Form(args) - } - - testAgent(t, handler, wrapAgent, "a=b") - - ReleaseArgs(args) -} - -type jsonData struct { - F string `json:"f"` } func Test_Client_Agent_Json(t *testing.T) { @@ -352,6 +366,30 @@ func Test_Client_Agent_Json_Error(t *testing.T) { utils.AssertEqual(t, "json: unsupported type: complex128", errs[0].Error()) } +func Test_Client_Agent_Form(t *testing.T) { + handler := func(c *Ctx) error { + utils.AssertEqual(t, MIMEApplicationForm, string(c.Request().Header.ContentType())) + + return c.Send(c.Request().Body()) + } + + args := AcquireArgs() + + args.Set("a", "b") + + wrapAgent := func(a *Agent) { + a.Form(args) + } + + testAgent(t, handler, wrapAgent, "a=b") + + ReleaseArgs(args) +} + +type jsonData struct { + F string `json:"f"` +} + func Test_Client_Debug(t *testing.T) { handler := func(c *Ctx) error { return c.SendString("debug") @@ -375,37 +413,6 @@ func Test_Client_Debug(t *testing.T) { utils.AssertEqual(t, true, strings.Contains(str, "Content-Type: text/plain; charset=utf-8\r\nContent-Length: 5\r\n\r\ndebug")) } -func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), excepted string, count ...int) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - app := New(Config{DisableStartupMessage: true}) - - app.Get("/", handler) - - go app.Listener(ln) //nolint:errcheck - - c := 1 - if len(count) > 0 { - c = count[0] - } - - for i := 0; i < c; i++ { - a := Get("http://example.com") - - wrapAgent(a) - - a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - - code, body, errs := a.String() - - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, excepted, body) - utils.AssertEqual(t, 0, len(errs)) - } -} - func Test_Client_Agent_Timeout(t *testing.T) { t.Parallel() @@ -432,6 +439,76 @@ func Test_Client_Agent_Timeout(t *testing.T) { utils.AssertEqual(t, "timeout", errs[0].Error()) } +func Test_Client_Agent_Reuse(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("reuse") + }) + + go app.Listener(ln) //nolint:errcheck + + a := Get("http://example.com"). + Reuse() + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "reuse", body) + utils.AssertEqual(t, 0, len(errs)) + + code, body, errs = a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "reuse", body) + utils.AssertEqual(t, 0, len(errs)) +} + +func Test_Client_Agent_TLS(t *testing.T) { + t.Parallel() + + // Create tls certificate + cer, err := tls.LoadX509KeyPair("./.github/testdata/ssl.pem", "./.github/testdata/ssl.key") + utils.AssertEqual(t, nil, err) + + config := &tls.Config{ + Certificates: []tls.Certificate{cer}, + } + + ln, err := net.Listen(NetworkTCP4, "127.0.0.1:0") + utils.AssertEqual(t, nil, err) + + ln = tls.NewListener(ln, config) + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("tls") + }) + + go app.Listener(ln) //nolint:errcheck + + code, body, errs := Get("https://" + ln.Addr().String()). + InsecureSkipVerify(). + TLSConfig(config). + InsecureSkipVerify(). + String() + + utils.AssertEqual(t, 0, len(errs)) + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "tls", body) +} + +type data struct { + Success bool `json:"success"` +} + func Test_Client_Agent_MaxRedirectsCount(t *testing.T) { t.Parallel() @@ -478,122 +555,6 @@ func Test_Client_Agent_MaxRedirectsCount(t *testing.T) { }) } -func Test_Client_Agent_Custom(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - app := New(Config{DisableStartupMessage: true}) - - app.Get("/", func(c *Ctx) error { - return c.SendString("custom") - }) - - go app.Listener(ln) //nolint:errcheck - - for i := 0; i < 5; i++ { - a := AcquireAgent() - req := AcquireRequest() - resp := AcquireResponse() - - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://example.com") - a.Request(req) - - utils.AssertEqual(t, nil, a.Parse()) - - a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - - code, body, errs := a.String(resp) - - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, "custom", body) - utils.AssertEqual(t, "custom", string(resp.Body())) - utils.AssertEqual(t, 0, len(errs)) - - ReleaseRequest(req) - ReleaseResponse(resp) - } -} - -func Test_Client_Agent_Reuse(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - app := New(Config{DisableStartupMessage: true}) - - app.Get("/", func(c *Ctx) error { - return c.SendString("reuse") - }) - - go app.Listener(ln) //nolint:errcheck - - a := Get("http://example.com"). - Reuse() - - a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - - code, body, errs := a.String() - - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, "reuse", body) - utils.AssertEqual(t, 0, len(errs)) - - code, body, errs = a.String() - - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, "reuse", body) - utils.AssertEqual(t, 0, len(errs)) -} - -func Test_Client_Agent_Parse(t *testing.T) { - t.Parallel() - - a := Get("https://example.com:10443") - - utils.AssertEqual(t, nil, a.Parse()) -} - -func Test_Client_Agent_TLS(t *testing.T) { - t.Parallel() - - // Create tls certificate - cer, err := tls.LoadX509KeyPair("./.github/testdata/ssl.pem", "./.github/testdata/ssl.key") - utils.AssertEqual(t, nil, err) - - config := &tls.Config{ - Certificates: []tls.Certificate{cer}, - } - - ln, err := net.Listen(NetworkTCP4, "127.0.0.1:0") - utils.AssertEqual(t, nil, err) - - ln = tls.NewListener(ln, config) - - app := New(Config{DisableStartupMessage: true}) - - app.Get("/", func(c *Ctx) error { - return c.SendString("tls") - }) - - go app.Listener(ln) //nolint:errcheck - - code, body, errs := Get("https://" + ln.Addr().String()). - InsecureSkipVerify(). - TLSConfig(config). - InsecureSkipVerify(). - String() - - utils.AssertEqual(t, 0, len(errs)) - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, "tls", body) -} - -type data struct { - Success bool `json:"success"` -} - func Test_Client_Agent_Struct(t *testing.T) { t.Parallel() @@ -642,7 +603,46 @@ func Test_Client_Agent_Struct(t *testing.T) { }) } +func Test_Client_Agent_Parse(t *testing.T) { + t.Parallel() + + a := Get("https://example.com:10443") + + utils.AssertEqual(t, nil, a.Parse()) +} + func Test_AddMissingPort_TLS(t *testing.T) { addr := addMissingPort("example.com", true) utils.AssertEqual(t, "example.com:443", addr) } + +func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), excepted string, count ...int) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", handler) + + go app.Listener(ln) //nolint:errcheck + + c := 1 + if len(count) > 0 { + c = count[0] + } + + for i := 0; i < c; i++ { + a := Get("http://example.com") + + wrapAgent(a) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, excepted, body) + utils.AssertEqual(t, 0, len(errs)) + } +} From 20104ba10f65fbb9e4bc62bad617264c0cce1d3c Mon Sep 17 00:00:00 2001 From: Kiyon Date: Fri, 19 Feb 2021 09:53:51 +0800 Subject: [PATCH 04/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20BasicAuth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 12 +++++++++--- client_test.go | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index c8e78d17..d6b7668b 100644 --- a/client.go +++ b/client.go @@ -234,6 +234,14 @@ func (a *Agent) QueryString(queryString string) *Agent { return a } +// BasicAuth sets URI username and password. +func (a *Agent) BasicAuth(username, password string) *Agent { + a.req.URI().SetUsername(username) + a.req.URI().SetPassword(password) + + return a +} + /************************** End URI Setting **************************/ /************************** Request Setting **************************/ @@ -289,9 +297,7 @@ func (a *Agent) Form(args *Args) *Agent { a.req.Header.SetContentType(MIMEApplicationForm) if args != nil { - if _, err := args.WriteTo(a.req.BodyWriter()); err != nil { - a.errs = append(a.errs, err) - } + a.req.SetBody(args.QueryString()) } return a diff --git a/client_test.go b/client_test.go index 1e952bcf..0f3e2a48 100644 --- a/client_test.go +++ b/client_test.go @@ -3,6 +3,7 @@ package fiber import ( "bytes" "crypto/tls" + "encoding/base64" "net" "strings" "testing" @@ -279,6 +280,24 @@ func Test_Client_Agent_QueryString(t *testing.T) { testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") } +func Test_Client_Agent_BasicAuth(t *testing.T) { + handler := func(c *Ctx) error { + // Get authorization header + auth := c.Get(HeaderAuthorization) + // Decode the header contents + raw, err := base64.StdEncoding.DecodeString(auth[6:]) + utils.AssertEqual(t, nil, err) + + return c.Send(raw) + } + + wrapAgent := func(a *Agent) { + a.BasicAuth("foo", "bar") + } + + testAgent(t, handler, wrapAgent, "foo:bar") +} + func Test_Client_Agent_BodyString(t *testing.T) { handler := func(c *Ctx) error { return c.Send(c.Request().Body()) From 5af0d42e7bfad73019cbf2a327166c28645ad34b Mon Sep 17 00:00:00 2001 From: Kiyon Date: Fri, 19 Feb 2021 10:03:51 +0800 Subject: [PATCH 05/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20XML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 18 ++++++++++++++++-- client_test.go | 43 ++++++++++++++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index d6b7668b..07b139bf 100644 --- a/client.go +++ b/client.go @@ -3,6 +3,7 @@ package fiber import ( "bytes" "crypto/tls" + "encoding/xml" "fmt" "io" "net" @@ -277,8 +278,8 @@ func (a *Agent) Request(req *Request) *Agent { return a } -// Json sends a json request. -func (a *Agent) Json(v interface{}) *Agent { +// JSON sends a JSON request. +func (a *Agent) JSON(v interface{}) *Agent { a.req.Header.SetContentType(MIMEApplicationJSON) if body, err := json.Marshal(v); err != nil { @@ -290,6 +291,19 @@ func (a *Agent) Json(v interface{}) *Agent { return a } +// XML sends a XML request. +func (a *Agent) XML(v interface{}) *Agent { + a.req.Header.SetContentType(MIMEApplicationXML) + + if body, err := xml.Marshal(v); err != nil { + a.errs = append(a.errs, err) + } else { + a.req.SetBody(body) + } + + return a +} + // Form sends request with body if args is non-nil. // // Note that this will force http method to post. diff --git a/client_test.go b/client_test.go index 0f3e2a48..55496b0d 100644 --- a/client_test.go +++ b/client_test.go @@ -368,15 +368,15 @@ func Test_Client_Agent_Json(t *testing.T) { } wrapAgent := func(a *Agent) { - a.Json(jsonData{F: "f"}) + a.JSON(data{Success: true}) } - testAgent(t, handler, wrapAgent, `{"f":"f"}`) + testAgent(t, handler, wrapAgent, `{"success":true}`) } func Test_Client_Agent_Json_Error(t *testing.T) { a := Get("http://example.com"). - Json(complex(1, 1)) + JSON(complex(1, 1)) _, body, errs := a.String() @@ -385,6 +385,31 @@ func Test_Client_Agent_Json_Error(t *testing.T) { utils.AssertEqual(t, "json: unsupported type: complex128", errs[0].Error()) } +func Test_Client_Agent_XML(t *testing.T) { + handler := func(c *Ctx) error { + utils.AssertEqual(t, MIMEApplicationXML, string(c.Request().Header.ContentType())) + + return c.Send(c.Request().Body()) + } + + wrapAgent := func(a *Agent) { + a.XML(data{Success: true}) + } + + testAgent(t, handler, wrapAgent, "true") +} + +func Test_Client_Agent_XML_Error(t *testing.T) { + a := Get("http://example.com"). + XML(complex(1, 1)) + + _, body, errs := a.String() + + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "xml: unsupported type: complex128", errs[0].Error()) +} + func Test_Client_Agent_Form(t *testing.T) { handler := func(c *Ctx) error { utils.AssertEqual(t, MIMEApplicationForm, string(c.Request().Header.ContentType())) @@ -405,10 +430,6 @@ func Test_Client_Agent_Form(t *testing.T) { ReleaseArgs(args) } -type jsonData struct { - F string `json:"f"` -} - func Test_Client_Debug(t *testing.T) { handler := func(c *Ctx) error { return c.SendString("debug") @@ -524,10 +545,6 @@ func Test_Client_Agent_TLS(t *testing.T) { utils.AssertEqual(t, "tls", body) } -type data struct { - Success bool `json:"success"` -} - func Test_Client_Agent_MaxRedirectsCount(t *testing.T) { t.Parallel() @@ -665,3 +682,7 @@ func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), exce utils.AssertEqual(t, 0, len(errs)) } } + +type data struct { + Success bool `json:"success" xml:"success"` +} From f4307905a4276a913ffe49d187ba88f171d7ad00 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Fri, 19 Feb 2021 17:21:59 +0800 Subject: [PATCH 06/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20MultipartForm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 153 ++++++++++++++++++++++++++++++++++++++++++++++++- client_test.go | 148 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 298 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 07b139bf..40493bc7 100644 --- a/client.go +++ b/client.go @@ -6,8 +6,11 @@ import ( "encoding/xml" "fmt" "io" + "io/ioutil" + "mime/multipart" "net" "os" + "path/filepath" "strconv" "strings" "sync" @@ -90,8 +93,10 @@ type Agent struct { args *Args timeout time.Duration errs []error + formFiles []*FormFile debugWriter io.Writer maxRedirectsCount int + boundary string Name string NoDefaultUserAgentHeader bool reuse bool @@ -306,7 +311,8 @@ func (a *Agent) XML(v interface{}) *Agent { // Form sends request with body if args is non-nil. // -// Note that this will force http method to post. +// It is recommended obtaining args via AcquireArgs +// in performance-critical code. func (a *Agent) Form(args *Args) *Agent { a.req.Header.SetContentType(MIMEApplicationForm) @@ -317,6 +323,118 @@ func (a *Agent) Form(args *Args) *Agent { return a } +// FormFile represents multipart form file +type FormFile struct { + // Fieldname is form file's field name + Fieldname string + // Name is form file's name + Name string + // Content is form file's content + Content []byte + // autoRelease indicates if returns the object + // acquired via AcquireFormFile to the pool. + autoRelease bool +} + +// FileData appends files for multipart form request. +// +// It is recommended obtaining formFile via AcquireFormFile +// in performance-critical code. +func (a *Agent) FileData(formFiles ...*FormFile) *Agent { + a.formFiles = append(a.formFiles, formFiles...) + + return a +} + +// SendFile reads file and appends it to multipart form request. +func (a *Agent) SendFile(filename string, fieldname ...string) *Agent { + content, err := ioutil.ReadFile(filename) + if err != nil { + a.errs = append(a.errs, err) + return a + } + + ff := AcquireFormFile() + if len(fieldname) > 0 && fieldname[0] != "" { + ff.Fieldname = fieldname[0] + } else { + ff.Fieldname = "file" + strconv.Itoa(len(a.formFiles)+1) + } + ff.Name = filepath.Base(filename) + ff.Content = append(ff.Content, content...) + ff.autoRelease = true + + a.formFiles = append(a.formFiles, ff) + + return a +} + +// SendFiles reads files and appends them to multipart form request. +// +// Examples: +// SendFile("/path/to/file1", "fieldname1", "/path/to/file2") +func (a *Agent) SendFiles(filenamesAndFieldnames ...string) *Agent { + pairs := len(filenamesAndFieldnames) + if pairs&1 == 1 { + filenamesAndFieldnames = append(filenamesAndFieldnames, "") + } + + for i := 0; i < pairs; i += 2 { + a.SendFile(filenamesAndFieldnames[i], filenamesAndFieldnames[i+1]) + } + + return a +} + +// Boundary sets boundary for multipart form request. +func (a *Agent) Boundary(boundary string) *Agent { + a.boundary = boundary + + return a +} + +// MultipartForm sends multipart form request with k-v and files. +// +// It is recommended obtaining args via AcquireArgs +// in performance-critical code. +func (a *Agent) MultipartForm(args *Args) *Agent { + mw := multipart.NewWriter(a.req.BodyWriter()) + + if a.boundary != "" { + if err := mw.SetBoundary(a.boundary); err != nil { + a.errs = append(a.errs, err) + return a + } + } + + a.req.Header.SetMultipartFormBoundary(mw.Boundary()) + + if args != nil { + args.VisitAll(func(key, value []byte) { + if err := mw.WriteField(getString(key), getString(value)); err != nil { + a.errs = append(a.errs, err) + } + }) + } + + for _, ff := range a.formFiles { + w, err := mw.CreateFormFile(ff.Fieldname, ff.Name) + if err != nil { + a.errs = append(a.errs, err) + continue + } + if _, err = w.Write(ff.Content); err != nil { + a.errs = append(a.errs, err) + } + } + + if err := mw.Close(); err != nil { + a.errs = append(a.errs, err) + } + + return a +} + /************************** End Request Setting **************************/ /************************** Agent Setting **************************/ @@ -479,8 +597,16 @@ func (a *Agent) reset() { a.reuse = false a.parsed = false a.maxRedirectsCount = 0 + a.boundary = "" a.Name = "" a.NoDefaultUserAgentHeader = false + for i, ff := range a.formFiles { + if ff.autoRelease { + ReleaseFormFile(ff) + } + a.formFiles[i] = nil + } + a.formFiles = a.formFiles[:0] } var ( @@ -489,6 +615,7 @@ var ( requestPool sync.Pool responsePool sync.Pool argsPool sync.Pool + formFilePool sync.Pool ) // AcquireAgent returns an empty Agent instance from createAgent pool. @@ -605,6 +732,30 @@ func ReleaseArgs(a *Args) { argsPool.Put(a) } +// AcquireFormFile returns an empty FormFile object from the pool. +// +// The returned FormFile may be returned to the pool with ReleaseFormFile +// when no longer needed. This allows reducing GC load. +func AcquireFormFile() *FormFile { + v := formFilePool.Get() + if v == nil { + return &FormFile{} + } + return v.(*FormFile) +} + +// ReleaseFormFile returns the object acquired via AcquireFormFile to the pool. +// +// String not access the released FormFile object, otherwise data races may occur. +func ReleaseFormFile(ff *FormFile) { + ff.Fieldname = "" + ff.Name = "" + ff.Content = ff.Content[:0] + ff.autoRelease = false + + formFilePool.Put(ff) +} + var ( strHTTP = []byte("http") strHTTPS = []byte("https") diff --git a/client_test.go b/client_test.go index 55496b0d..64bb0a3f 100644 --- a/client_test.go +++ b/client_test.go @@ -4,7 +4,11 @@ import ( "bytes" "crypto/tls" "encoding/base64" + "io/ioutil" + "mime/multipart" "net" + "path/filepath" + "regexp" "strings" "testing" "time" @@ -419,17 +423,157 @@ func Test_Client_Agent_Form(t *testing.T) { args := AcquireArgs() - args.Set("a", "b") + args.Set("foo", "bar") wrapAgent := func(a *Agent) { a.Form(args) } - testAgent(t, handler, wrapAgent, "a=b") + testAgent(t, handler, wrapAgent, "foo=bar") ReleaseArgs(args) } +func Test_Client_Agent_Multipart(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Post("/", func(c *Ctx) error { + utils.AssertEqual(t, "multipart/form-data; boundary=myBoundary", c.Get(HeaderContentType)) + + mf, err := c.MultipartForm() + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, "bar", mf.Value["foo"][0]) + + return c.Send(c.Request().Body()) + }) + + go app.Listener(ln) //nolint:errcheck + + args := AcquireArgs() + + args.Set("foo", "bar") + + a := Post("http://example.com"). + Boundary("myBoundary"). + MultipartForm(args) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "--myBoundary\r\nContent-Disposition: form-data; name=\"foo\"\r\n\r\nbar\r\n--myBoundary--\r\n", body) + utils.AssertEqual(t, 0, len(errs)) + ReleaseArgs(args) +} + +func Test_Client_Agent_Multipart_SendFiles(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Post("/", func(c *Ctx) error { + utils.AssertEqual(t, "multipart/form-data; boundary=myBoundary", c.Get(HeaderContentType)) + + mf, err := c.MultipartForm() + utils.AssertEqual(t, nil, err) + + fh1 := mf.File["field1"][0] + utils.AssertEqual(t, fh1.Filename, "name") + buf := make([]byte, fh1.Size, fh1.Size) + f, err := fh1.Open() + utils.AssertEqual(t, nil, err) + defer func() { _ = f.Close() }() + _, err = f.Read(buf) + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, "form file", string(buf)) + + checkFormFile(t, mf.File["index"][0], ".github/testdata/index.html") + checkFormFile(t, mf.File["file3"][0], ".github/testdata/index.tmpl") + + return c.SendString("multipart form files") + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + ff := AcquireFormFile() + ff.Fieldname = "field1" + ff.Name = "name" + ff.Content = []byte("form file") + + a := Post("http://example.com"). + Boundary("myBoundary"). + FileData(ff). + SendFiles(".github/testdata/index.html", "index", ".github/testdata/index.tmpl"). + MultipartForm(nil) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "multipart form files", body) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseFormFile(ff) + } +} + +func checkFormFile(t *testing.T, fh *multipart.FileHeader, filename string) { + basename := filepath.Base(filename) + utils.AssertEqual(t, fh.Filename, basename) + + b1, err := ioutil.ReadFile(filename) + utils.AssertEqual(t, nil, err) + + b2 := make([]byte, fh.Size, fh.Size) + f, err := fh.Open() + utils.AssertEqual(t, nil, err) + defer func() { _ = f.Close() }() + _, err = f.Read(b2) + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, b1, b2) +} + +func Test_Client_Agent_Multipart_Random_Boundary(t *testing.T) { + t.Parallel() + + a := Post("http://example.com"). + MultipartForm(nil) + + reg := regexp.MustCompile(`multipart/form-data; boundary=\w{30}`) + + utils.AssertEqual(t, true, reg.Match(a.req.Header.Peek(HeaderContentType))) +} + +func Test_Client_Agent_Multipart_Invalid_Boundary(t *testing.T) { + t.Parallel() + + a := Post("http://example.com"). + Boundary("*"). + MultipartForm(nil) + + utils.AssertEqual(t, 1, len(a.errs)) + utils.AssertEqual(t, "mime: invalid boundary character", a.errs[0].Error()) +} + +func Test_Client_Agent_SendFile_Error(t *testing.T) { + t.Parallel() + + a := Post("http://example.com"). + SendFile("", "") + + utils.AssertEqual(t, 1, len(a.errs)) + utils.AssertEqual(t, "open : no such file or directory", a.errs[0].Error()) +} + func Test_Client_Debug(t *testing.T) { handler := func(c *Ctx) error { return c.SendString("debug") From 5093eb8e276fe11187ae7b1a2d7cded402406e3c Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 14:31:17 +0800 Subject: [PATCH 07/24] =?UTF-8?q?=F0=9F=91=B7=20Fix=20G304=20and=20allow?= =?UTF-8?q?=20G402?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client.go b/client.go index 40493bc7..70556b9c 100644 --- a/client.go +++ b/client.go @@ -348,7 +348,7 @@ func (a *Agent) FileData(formFiles ...*FormFile) *Agent { // SendFile reads file and appends it to multipart form request. func (a *Agent) SendFile(filename string, fieldname ...string) *Agent { - content, err := ioutil.ReadFile(filename) + content, err := ioutil.ReadFile(filepath.Clean(filename)) if err != nil { a.errs = append(a.errs, err) return a @@ -467,8 +467,10 @@ func (a *Agent) Reuse() *Agent { // certificate chain and host name. func (a *Agent) InsecureSkipVerify() *Agent { if a.HostClient.TLSConfig == nil { + /* #nosec G402 */ a.HostClient.TLSConfig = &tls.Config{InsecureSkipVerify: true} } else { + /* #nosec G402 */ a.HostClient.TLSConfig.InsecureSkipVerify = true } From 7c5fac67cf796dfb6fef25ea13f50d4a6e92ab8c Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 14:31:48 +0800 Subject: [PATCH 08/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 55 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/client.go b/client.go index 70556b9c..c92660c1 100644 --- a/client.go +++ b/client.go @@ -26,6 +26,7 @@ import ( // and use CopyTo instead. // // Request instance MUST NOT be used from concurrently running goroutines. +// Copy from fasthttp type Request = fasthttp.Request // Response represents HTTP response. @@ -34,6 +35,7 @@ type Request = fasthttp.Request // and use CopyTo instead. // // Response instance MUST NOT be used from concurrently running goroutines. +// Copy from fasthttp type Response = fasthttp.Response // Args represents query arguments. @@ -42,6 +44,7 @@ type Response = fasthttp.Response // and use CopyTo(). // // Args instance MUST NOT be used from concurrently running goroutines. +// Copy from fasthttp type Args = fasthttp.Args var defaultClient Client @@ -50,7 +53,11 @@ var defaultClient Client // // It is safe calling Client methods from concurrently running goroutines. type Client struct { - UserAgent string + // UserAgent is used in User-Agent request header. + UserAgent string + + // NoDefaultUserAgentHeader when set to true, causes the default + // User-Agent header to be excluded from the Request. NoDefaultUserAgentHeader bool } @@ -620,28 +627,6 @@ var ( formFilePool sync.Pool ) -// AcquireAgent returns an empty Agent instance from createAgent pool. -// -// The returned Agent instance may be passed to ReleaseAgent when it is -// no longer needed. This allows Agent recycling, reduces GC pressure -// and usually improves performance. -func AcquireAgent() *Agent { - v := agentPool.Get() - if v == nil { - return &Agent{req: fasthttp.AcquireRequest()} - } - return v.(*Agent) -} - -// ReleaseAgent returns a acquired via AcquireAgent to createAgent pool. -// -// It is forbidden accessing req and/or its' members after returning -// it to createAgent pool. -func ReleaseAgent(a *Agent) { - a.reset() - agentPool.Put(a) -} - // AcquireClient returns an empty Client instance from client pool. // // The returned Client instance may be passed to ReleaseClient when it is @@ -666,11 +651,34 @@ func ReleaseClient(c *Client) { clientPool.Put(c) } +// AcquireAgent returns an empty Agent instance from createAgent pool. +// +// The returned Agent instance may be passed to ReleaseAgent when it is +// no longer needed. This allows Agent recycling, reduces GC pressure +// and usually improves performance. +func AcquireAgent() *Agent { + v := agentPool.Get() + if v == nil { + return &Agent{req: fasthttp.AcquireRequest()} + } + return v.(*Agent) +} + +// ReleaseAgent returns a acquired via AcquireAgent to createAgent pool. +// +// It is forbidden accessing req and/or its' members after returning +// it to createAgent pool. +func ReleaseAgent(a *Agent) { + a.reset() + agentPool.Put(a) +} + // AcquireRequest returns an empty Request instance from request pool. // // The returned Request instance may be passed to ReleaseRequest when it is // no longer needed. This allows Request recycling, reduces GC pressure // and usually improves performance. +// Copy from fasthttp func AcquireRequest() *Request { v := requestPool.Get() if v == nil { @@ -683,6 +691,7 @@ func AcquireRequest() *Request { // // It is forbidden accessing req and/or its' members after returning // it to request pool. +// Copy from fasthttp func ReleaseRequest(req *Request) { req.Reset() requestPool.Put(req) From 645e01181394dea5afb1ded476497d9443bf1175 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 14:50:20 +0800 Subject: [PATCH 09/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20HEAD=20PUT=20PATCH?= =?UTF-8?q?=20DELETE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 32 +++++++++++++ client_test.go | 123 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/client.go b/client.go index c92660c1..227f43a4 100644 --- a/client.go +++ b/client.go @@ -69,6 +69,14 @@ func (c *Client) Get(url string) *Agent { return c.createAgent(MethodGet, url) } +// Head returns a agent with http method HEAD. +func Head(url string) *Agent { return defaultClient.Head(url) } + +// Head returns a agent with http method GET. +func (c *Client) Head(url string) *Agent { + return c.createAgent(MethodHead, url) +} + // Post sends POST request to the given url. func Post(url string) *Agent { return defaultClient.Post(url) } @@ -77,6 +85,30 @@ func (c *Client) Post(url string) *Agent { return c.createAgent(MethodPost, url) } +// Put sends PUT request to the given url. +func Put(url string) *Agent { return defaultClient.Put(url) } + +// Put sends PUT request to the given url. +func (c *Client) Put(url string) *Agent { + return c.createAgent(MethodPut, url) +} + +// Patch sends PATCH request to the given url. +func Patch(url string) *Agent { return defaultClient.Patch(url) } + +// Patch sends PATCH request to the given url. +func (c *Client) Patch(url string) *Agent { + return c.createAgent(MethodPatch, url) +} + +// Delete sends DELETE request to the given url. +func Delete(url string) *Agent { return defaultClient.Delete(url) } + +// Delete sends DELETE request to the given url. +func (c *Client) Delete(url string) *Agent { + return c.createAgent(MethodDelete, url) +} + func (c *Client) createAgent(method, url string) *Agent { a := AcquireAgent() a.req.Header.SetMethod(method) diff --git a/client_test.go b/client_test.go index 64bb0a3f..1781762c 100644 --- a/client_test.go +++ b/client_test.go @@ -80,6 +80,32 @@ func Test_Client_Get(t *testing.T) { } } +func Test_Client_Head(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString(c.Hostname()) + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + a := Head("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 0, len(errs)) + } +} + func Test_Client_Post(t *testing.T) { t.Parallel() @@ -113,6 +139,103 @@ func Test_Client_Post(t *testing.T) { } } +func Test_Client_Put(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Put("/", func(c *Ctx) error { + return c.SendString(c.FormValue("foo")) + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + args := AcquireArgs() + + args.Set("foo", "bar") + + a := Put("http://example.com"). + Form(args) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "bar", body) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseArgs(args) + } +} + +func Test_Client_Patch(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Patch("/", func(c *Ctx) error { + return c.SendString(c.FormValue("foo")) + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + args := AcquireArgs() + + args.Set("foo", "bar") + + a := Patch("http://example.com"). + Form(args) + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "bar", body) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseArgs(args) + } +} + +func Test_Client_Delete(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Delete("/", func(c *Ctx) error { + return c.Status(StatusNoContent). + SendString("deleted") + }) + + go app.Listener(ln) //nolint:errcheck + + for i := 0; i < 5; i++ { + args := AcquireArgs() + + a := Delete("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.String() + + utils.AssertEqual(t, StatusNoContent, code) + utils.AssertEqual(t, "", body) + utils.AssertEqual(t, 0, len(errs)) + + ReleaseArgs(args) + } +} + func Test_Client_UserAgent(t *testing.T) { t.Parallel() From 040ab0e101555dcc5f7d8f90dd26428965b89031 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 14:50:37 +0800 Subject: [PATCH 10/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20test=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/client_test.go b/client_test.go index 1781762c..1288d92c 100644 --- a/client_test.go +++ b/client_test.go @@ -114,7 +114,8 @@ func Test_Client_Post(t *testing.T) { app := New(Config{DisableStartupMessage: true}) app.Post("/", func(c *Ctx) error { - return c.SendString(c.Hostname()) + return c.Status(StatusCreated). + SendString(c.FormValue("foo")) }) go app.Listener(ln) //nolint:errcheck @@ -131,8 +132,8 @@ func Test_Client_Post(t *testing.T) { code, body, errs := a.String() - utils.AssertEqual(t, StatusOK, code) - utils.AssertEqual(t, "example.com", body) + utils.AssertEqual(t, StatusCreated, code) + utils.AssertEqual(t, "bar", body) utils.AssertEqual(t, 0, len(errs)) ReleaseArgs(args) @@ -604,10 +605,8 @@ func Test_Client_Agent_Multipart_SendFiles(t *testing.T) { app.Post("/", func(c *Ctx) error { utils.AssertEqual(t, "multipart/form-data; boundary=myBoundary", c.Get(HeaderContentType)) - mf, err := c.MultipartForm() + fh1, err := c.FormFile("field1") utils.AssertEqual(t, nil, err) - - fh1 := mf.File["field1"][0] utils.AssertEqual(t, fh1.Filename, "name") buf := make([]byte, fh1.Size, fh1.Size) f, err := fh1.Open() @@ -617,8 +616,13 @@ func Test_Client_Agent_Multipart_SendFiles(t *testing.T) { utils.AssertEqual(t, nil, err) utils.AssertEqual(t, "form file", string(buf)) - checkFormFile(t, mf.File["index"][0], ".github/testdata/index.html") - checkFormFile(t, mf.File["file3"][0], ".github/testdata/index.tmpl") + fh2, err := c.FormFile("index") + utils.AssertEqual(t, nil, err) + checkFormFile(t, fh2, ".github/testdata/index.html") + + fh3, err := c.FormFile("file3") + utils.AssertEqual(t, nil, err) + checkFormFile(t, fh3, ".github/testdata/index.tmpl") return c.SendString("multipart form files") }) @@ -691,10 +695,10 @@ func Test_Client_Agent_SendFile_Error(t *testing.T) { t.Parallel() a := Post("http://example.com"). - SendFile("", "") + SendFile("non-exist-file!", "") utils.AssertEqual(t, 1, len(a.errs)) - utils.AssertEqual(t, "open : no such file or directory", a.errs[0].Error()) + utils.AssertEqual(t, "open non-exist-file!: no such file or directory", a.errs[0].Error()) } func Test_Client_Debug(t *testing.T) { From a60d23343c5143749a4a62bca5c287da01fa024d Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 14:56:47 +0800 Subject: [PATCH 11/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20test=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client_test.go b/client_test.go index 1288d92c..6108a123 100644 --- a/client_test.go +++ b/client_test.go @@ -698,7 +698,7 @@ func Test_Client_Agent_SendFile_Error(t *testing.T) { SendFile("non-exist-file!", "") utils.AssertEqual(t, 1, len(a.errs)) - utils.AssertEqual(t, "open non-exist-file!: no such file or directory", a.errs[0].Error()) + utils.AssertEqual(t, true, strings.Contains(a.errs[0].Error(), "open non-exist-file!")) } func Test_Client_Debug(t *testing.T) { From 62d311133bb983861d6d7d23dbd327e7b2bbacf0 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 15:46:41 +0800 Subject: [PATCH 12/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20Bytes=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 130 +++++++++++++++++++++++++++++++++++++++++++++++++ client_test.go | 52 +++++++++++++++----- 2 files changed, 169 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 227f43a4..07bb46b3 100644 --- a/client.go +++ b/client.go @@ -207,6 +207,33 @@ func (a *Agent) Set(k, v string) *Agent { return a } +// SetBytesK sets the given 'key: value' header. +// +// Use AddBytesK for setting multiple header values under the same key. +func (a *Agent) SetBytesK(k []byte, v string) *Agent { + a.req.Header.SetBytesK(k, v) + + return a +} + +// SetBytesV sets the given 'key: value' header. +// +// Use AddBytesV for setting multiple header values under the same key. +func (a *Agent) SetBytesV(k string, v []byte) *Agent { + a.req.Header.SetBytesV(k, v) + + return a +} + +// SetBytesKV sets the given 'key: value' header. +// +// Use AddBytesKV for setting multiple header values under the same key. +func (a *Agent) SetBytesKV(k []byte, v []byte) *Agent { + a.req.Header.SetBytesKV(k, v) + + return a +} + // Add adds the given 'key: value' header. // // Multiple headers with the same key may be added with this function. @@ -217,6 +244,36 @@ func (a *Agent) Add(k, v string) *Agent { return a } +// AddBytesK adds the given 'key: value' header. +// +// Multiple headers with the same key may be added with this function. +// Use SetBytesK for setting a single header for the given key. +func (a *Agent) AddBytesK(k []byte, v string) *Agent { + a.req.Header.AddBytesK(k, v) + + return a +} + +// AddBytesV adds the given 'key: value' header. +// +// Multiple headers with the same key may be added with this function. +// Use SetBytesV for setting a single header for the given key. +func (a *Agent) AddBytesV(k string, v []byte) *Agent { + a.req.Header.AddBytesV(k, v) + + return a +} + +// AddBytesKV adds the given 'key: value' header. +// +// Multiple headers with the same key may be added with this function. +// Use SetBytesKV for setting a single header for the given key. +func (a *Agent) AddBytesKV(k []byte, v []byte) *Agent { + a.req.Header.AddBytesKV(k, v) + + return a +} + // ConnectionClose sets 'Connection: close' header. func (a *Agent) ConnectionClose() *Agent { a.req.Header.SetConnectionClose() @@ -231,6 +288,13 @@ func (a *Agent) UserAgent(userAgent string) *Agent { return a } +// UserAgentBytes sets User-Agent header value. +func (a *Agent) UserAgentBytes(userAgent []byte) *Agent { + a.req.Header.SetUserAgentBytes(userAgent) + + return a +} + // Cookie sets one 'key: value' cookie. func (a *Agent) Cookie(key, value string) *Agent { a.req.Header.SetCookie(key, value) @@ -238,6 +302,20 @@ func (a *Agent) Cookie(key, value string) *Agent { return a } +// CookieBytesK sets one 'key: value' cookie. +func (a *Agent) CookieBytesK(key []byte, value string) *Agent { + a.req.Header.SetCookieBytesK(key, value) + + return a +} + +// CookieBytesKV sets one 'key: value' cookie. +func (a *Agent) CookieBytesKV(key, value []byte) *Agent { + a.req.Header.SetCookieBytesKV(key, value) + + return a +} + // Cookies sets multiple 'key: value' cookies. func (a *Agent) Cookies(kv ...string) *Agent { for i := 1; i < len(kv); i += 2 { @@ -247,6 +325,15 @@ func (a *Agent) Cookies(kv ...string) *Agent { return a } +// CookiesBytesKV sets multiple 'key: value' cookies. +func (a *Agent) CookiesBytesKV(kv ...[]byte) *Agent { + for i := 1; i < len(kv); i += 2 { + a.req.Header.SetCookieBytesKV(kv[i-1], kv[i]) + } + + return a +} + // Referer sets Referer header value. func (a *Agent) Referer(referer string) *Agent { a.req.Header.SetReferer(referer) @@ -254,6 +341,13 @@ func (a *Agent) Referer(referer string) *Agent { return a } +// RefererBytes sets Referer header value. +func (a *Agent) RefererBytes(referer []byte) *Agent { + a.req.Header.SetRefererBytes(referer) + + return a +} + // ContentType sets Content-Type header value. func (a *Agent) ContentType(contentType string) *Agent { a.req.Header.SetContentType(contentType) @@ -261,6 +355,13 @@ func (a *Agent) ContentType(contentType string) *Agent { return a } +// ContentTypeBytes sets Content-Type header value. +func (a *Agent) ContentTypeBytes(contentType []byte) *Agent { + a.req.Header.SetContentTypeBytes(contentType) + + return a +} + /************************** End Header Setting **************************/ /************************** URI Setting **************************/ @@ -272,6 +373,13 @@ func (a *Agent) Host(host string) *Agent { return a } +// HostBytes sets host for the uri. +func (a *Agent) HostBytes(host []byte) *Agent { + a.req.URI().SetHostBytes(host) + + return a +} + // QueryString sets URI query string. func (a *Agent) QueryString(queryString string) *Agent { a.req.URI().SetQueryString(queryString) @@ -279,6 +387,13 @@ func (a *Agent) QueryString(queryString string) *Agent { return a } +// QueryStringBytes sets URI query string. +func (a *Agent) QueryStringBytes(queryString []byte) *Agent { + a.req.URI().SetQueryStringBytes(queryString) + + return a +} + // BasicAuth sets URI username and password. func (a *Agent) BasicAuth(username, password string) *Agent { a.req.URI().SetUsername(username) @@ -287,6 +402,14 @@ func (a *Agent) BasicAuth(username, password string) *Agent { return a } +// BasicAuthBytes sets URI username and password. +func (a *Agent) BasicAuthBytes(username, password []byte) *Agent { + a.req.URI().SetUsernameBytes(username) + a.req.URI().SetPasswordBytes(password) + + return a +} + /************************** End URI Setting **************************/ /************************** Request Setting **************************/ @@ -298,6 +421,13 @@ func (a *Agent) BodyString(bodyString string) *Agent { return a } +// Body sets request body. +func (a *Agent) Body(body []byte) *Agent { + a.req.SetBody(body) + + return a +} + // BodyStream sets request body stream and, optionally body size. // // If bodySize is >= 0, then the bodyStream must provide exactly bodySize bytes diff --git a/client_test.go b/client_test.go index 6108a123..c28d2954 100644 --- a/client_test.go +++ b/client_test.go @@ -283,7 +283,7 @@ func Test_Client_UserAgent(t *testing.T) { }) } -func Test_Client_Agent_Headers(t *testing.T) { +func Test_Client_Agent_Set_Or_Add_Headers(t *testing.T) { handler := func(c *Ctx) error { c.Request().Header.VisitAll(func(key, value []byte) { if k := string(key); k == "K1" || k == "K2" { @@ -296,11 +296,17 @@ func Test_Client_Agent_Headers(t *testing.T) { wrapAgent := func(a *Agent) { a.Set("k1", "v1"). - Add("k1", "v11"). - Set("k2", "v2") + SetBytesK([]byte("k1"), "v1"). + SetBytesV("k1", []byte("v1")). + AddBytesK([]byte("k1"), "v11"). + AddBytesV("k1", []byte("v22")). + AddBytesKV([]byte("k1"), []byte("v33")). + SetBytesKV([]byte("k2"), []byte("v2")). + Add("k2", "v22") + } - testAgent(t, handler, wrapAgent, "K1v1K1v11K2v2") + testAgent(t, handler, wrapAgent, "K1v1K1v11K1v22K1v33K2v2K2v22") } func Test_Client_Agent_Connection_Close(t *testing.T) { @@ -324,7 +330,8 @@ func Test_Client_Agent_UserAgent(t *testing.T) { } wrapAgent := func(a *Agent) { - a.UserAgent("ua") + a.UserAgent("ua"). + UserAgentBytes([]byte("ua")) } testAgent(t, handler, wrapAgent, "ua") @@ -338,8 +345,10 @@ func Test_Client_Agent_Cookie(t *testing.T) { wrapAgent := func(a *Agent) { a.Cookie("k1", "v1"). - Cookie("k2", "v2"). - Cookies("k3", "v3", "k4", "v4") + CookieBytesK([]byte("k2"), "v2"). + CookieBytesKV([]byte("k2"), []byte("v2")). + Cookies("k3", "v3", "k4", "v4"). + CookiesBytesKV([]byte("k3"), []byte("v3"), []byte("k4"), []byte("v4")) } testAgent(t, handler, wrapAgent, "v1v2v3v4") @@ -351,7 +360,8 @@ func Test_Client_Agent_Referer(t *testing.T) { } wrapAgent := func(a *Agent) { - a.Referer("http://referer.com") + a.Referer("http://referer.com"). + RefererBytes([]byte("http://referer.com")) } testAgent(t, handler, wrapAgent, "http://referer.com") @@ -363,13 +373,14 @@ func Test_Client_Agent_ContentType(t *testing.T) { } wrapAgent := func(a *Agent) { - a.ContentType("custom-type") + a.ContentType("custom-type"). + ContentTypeBytes([]byte("custom-type")) } testAgent(t, handler, wrapAgent, "custom-type") } -func Test_Client_Agent_Specific_Host(t *testing.T) { +func Test_Client_Agent_Host(t *testing.T) { t.Parallel() ln := fasthttputil.NewInmemoryListener() @@ -383,7 +394,8 @@ func Test_Client_Agent_Specific_Host(t *testing.T) { go app.Listener(ln) //nolint:errcheck a := Get("http://1.1.1.1:8080"). - Host("example.com") + Host("example.com"). + HostBytes([]byte("example.com")) utils.AssertEqual(t, "1.1.1.1:8080", a.HostClient.Addr) @@ -402,7 +414,8 @@ func Test_Client_Agent_QueryString(t *testing.T) { } wrapAgent := func(a *Agent) { - a.QueryString("foo=bar&bar=baz") + a.QueryString("foo=bar&bar=baz"). + QueryStringBytes([]byte("foo=bar&bar=baz")) } testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") @@ -420,7 +433,8 @@ func Test_Client_Agent_BasicAuth(t *testing.T) { } wrapAgent := func(a *Agent) { - a.BasicAuth("foo", "bar") + a.BasicAuth("foo", "bar"). + BasicAuthBytes([]byte("foo"), []byte("bar")) } testAgent(t, handler, wrapAgent, "foo:bar") @@ -438,6 +452,18 @@ func Test_Client_Agent_BodyString(t *testing.T) { testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") } +func Test_Client_Agent_Body(t *testing.T) { + handler := func(c *Ctx) error { + return c.Send(c.Request().Body()) + } + + wrapAgent := func(a *Agent) { + a.Body([]byte("foo=bar&bar=baz")) + } + + testAgent(t, handler, wrapAgent, "foo=bar&bar=baz") +} + func Test_Client_Agent_BodyStream(t *testing.T) { handler := func(c *Ctx) error { return c.Send(c.Request().Body()) From c477128e5b82f8ba6af8a00b0b2dd1aa00fcad35 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 16:12:06 +0800 Subject: [PATCH 13/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20test=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 29 ++++++++++++++++++----------- client_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 07bb46b3..f8155576 100644 --- a/client.go +++ b/client.go @@ -134,6 +134,7 @@ type Agent struct { errs []error formFiles []*FormFile debugWriter io.Writer + mw multipartWriter maxRedirectsCount int boundary string Name string @@ -142,8 +143,6 @@ type Agent struct { parsed bool } -var ErrorInvalidURI = fasthttp.ErrorInvalidURI - // Parse initializes URI and HostClient. func (a *Agent) Parse() error { if a.parsed { @@ -157,9 +156,6 @@ func (a *Agent) Parse() error { } uri := req.URI() - if uri == nil { - return ErrorInvalidURI - } isTLS := false scheme := uri.Scheme() @@ -567,27 +563,29 @@ func (a *Agent) Boundary(boundary string) *Agent { // It is recommended obtaining args via AcquireArgs // in performance-critical code. func (a *Agent) MultipartForm(args *Args) *Agent { - mw := multipart.NewWriter(a.req.BodyWriter()) + if a.mw == nil { + a.mw = multipart.NewWriter(a.req.BodyWriter()) + } if a.boundary != "" { - if err := mw.SetBoundary(a.boundary); err != nil { + if err := a.mw.SetBoundary(a.boundary); err != nil { a.errs = append(a.errs, err) return a } } - a.req.Header.SetMultipartFormBoundary(mw.Boundary()) + a.req.Header.SetMultipartFormBoundary(a.mw.Boundary()) if args != nil { args.VisitAll(func(key, value []byte) { - if err := mw.WriteField(getString(key), getString(value)); err != nil { + if err := a.mw.WriteField(getString(key), getString(value)); err != nil { a.errs = append(a.errs, err) } }) } for _, ff := range a.formFiles { - w, err := mw.CreateFormFile(ff.Fieldname, ff.Name) + w, err := a.mw.CreateFormFile(ff.Fieldname, ff.Name) if err != nil { a.errs = append(a.errs, err) continue @@ -597,7 +595,7 @@ func (a *Agent) MultipartForm(args *Args) *Agent { } } - if err := mw.Close(); err != nil { + if err := a.mw.Close(); err != nil { a.errs = append(a.errs, err) } @@ -765,6 +763,7 @@ func (a *Agent) reset() { a.args = nil a.errs = a.errs[:0] a.debugWriter = nil + a.mw = nil a.reuse = false a.parsed = false a.maxRedirectsCount = 0 @@ -934,3 +933,11 @@ var ( strHTTPS = []byte("https") defaultUserAgent = "fiber" ) + +type multipartWriter interface { + Boundary() string + SetBoundary(boundary string) error + CreateFormFile(fieldname, filename string) (io.Writer, error) + WriteField(fieldname, value string) error + Close() error +} diff --git a/client_test.go b/client_test.go index c28d2954..424d69bc 100644 --- a/client_test.go +++ b/client_test.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/tls" "encoding/base64" + "errors" + "io" "io/ioutil" "mime/multipart" "net" @@ -584,7 +586,7 @@ func Test_Client_Agent_Form(t *testing.T) { ReleaseArgs(args) } -func Test_Client_Agent_Multipart(t *testing.T) { +func Test_Client_Agent_MultipartForm(t *testing.T) { t.Parallel() ln := fasthttputil.NewInmemoryListener() @@ -621,7 +623,25 @@ func Test_Client_Agent_Multipart(t *testing.T) { ReleaseArgs(args) } -func Test_Client_Agent_Multipart_SendFiles(t *testing.T) { +func Test_Client_Agent_MultipartForm_Errors(t *testing.T) { + t.Parallel() + + a := AcquireAgent() + a.mw = &errorMultipartWriter{} + + args := AcquireArgs() + args.Set("foo", "bar") + + ff1 := &FormFile{"", "name1", []byte("content"), false} + ff2 := &FormFile{"", "name2", []byte("content"), false} + a.FileData(ff1, ff2). + MultipartForm(args) + + utils.AssertEqual(t, 4, len(a.errs)) + ReleaseArgs(args) +} + +func Test_Client_Agent_MultipartForm_SendFiles(t *testing.T) { t.Parallel() ln := fasthttputil.NewInmemoryListener() @@ -983,3 +1003,23 @@ func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), exce type data struct { Success bool `json:"success" xml:"success"` } + +type errorMultipartWriter struct { + count int +} + +func (e *errorMultipartWriter) Boundary() string { return "myBoundary" } +func (e *errorMultipartWriter) SetBoundary(_ string) error { return nil } +func (e *errorMultipartWriter) CreateFormFile(_, _ string) (io.Writer, error) { + if e.count == 0 { + e.count++ + return nil, errors.New("CreateFormFile error") + } + return errorWriter{}, nil +} +func (e *errorMultipartWriter) WriteField(_, _ string) error { return errors.New("WriteField error") } +func (e *errorMultipartWriter) Close() error { return errors.New("Close error") } + +type errorWriter struct{} + +func (errorWriter) Write(_ []byte) (int, error) { return 0, errors.New("Write error") } From c34ca83c064dac867d109cad5372d0ef381aff32 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Sat, 20 Feb 2021 16:45:13 +0800 Subject: [PATCH 14/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20JSONEncoder=20and=20?= =?UTF-8?q?JSONDecoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 44 ++++++++++++++++++++++++++++++++++++-- client_test.go | 6 +++++- utils/json.go | 9 ++++++++ utils/json_marshal.go | 5 ----- utils/json_marshal_test.go | 26 ---------------------- utils/json_test.go | 41 +++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 34 deletions(-) create mode 100644 utils/json.go delete mode 100644 utils/json_marshal.go delete mode 100644 utils/json_marshal_test.go create mode 100644 utils/json_test.go diff --git a/client.go b/client.go index f8155576..3db4cb6f 100644 --- a/client.go +++ b/client.go @@ -16,6 +16,8 @@ import ( "sync" "time" + "github.com/gofiber/fiber/v2/utils" + "github.com/gofiber/fiber/v2/internal/encoding/json" "github.com/valyala/fasthttp" ) @@ -59,6 +61,18 @@ type Client struct { // NoDefaultUserAgentHeader when set to true, causes the default // User-Agent header to be excluded from the Request. NoDefaultUserAgentHeader bool + + // When set by an external client of Fiber it will use the provided implementation of a + // JSONMarshal + // + // Allowing for flexibility in using another json library for encoding + JSONEncoder utils.JSONMarshal + + // When set by an external client of Fiber it will use the provided implementation of a + // JSONUnmarshal + // + // Allowing for flexibility in using another json library for decoding + JSONDecoder utils.JSONUnmarshal } // Get returns a agent with http method GET. @@ -116,6 +130,8 @@ func (c *Client) createAgent(method, url string) *Agent { a.Name = c.UserAgent a.NoDefaultUserAgentHeader = c.NoDefaultUserAgentHeader + a.jsonDecoder = c.JSONDecoder + a.jsonEncoder = c.JSONEncoder if err := a.Parse(); err != nil { a.errs = append(a.errs, err) @@ -135,6 +151,8 @@ type Agent struct { formFiles []*FormFile debugWriter io.Writer mw multipartWriter + jsonEncoder utils.JSONMarshal + jsonDecoder utils.JSONUnmarshal maxRedirectsCount int boundary string Name string @@ -450,9 +468,13 @@ func (a *Agent) Request(req *Request) *Agent { // JSON sends a JSON request. func (a *Agent) JSON(v interface{}) *Agent { + if a.jsonEncoder == nil { + a.jsonEncoder = json.Marshal + } + a.req.Header.SetContentType(MIMEApplicationJSON) - if body, err := json.Marshal(v); err != nil { + if body, err := a.jsonEncoder(v); err != nil { a.errs = append(a.errs, err) } else { a.req.SetBody(body) @@ -658,6 +680,20 @@ func (a *Agent) MaxRedirectsCount(count int) *Agent { return a } +// JSONEncoder sets custom json encoder. +func (a *Agent) JSONEncoder(jsonEncoder utils.JSONMarshal) *Agent { + a.jsonEncoder = jsonEncoder + + return a +} + +// JSONDecoder sets custom json decoder. +func (a *Agent) JSONDecoder(jsonDecoder utils.JSONUnmarshal) *Agent { + a.jsonDecoder = jsonDecoder + + return a +} + /************************** End Agent Setting **************************/ // Bytes returns the status code, bytes body and errors of url. @@ -738,9 +774,13 @@ func (a *Agent) String(resp ...*Response) (int, string, []error) { // Struct returns the status code, bytes body and errors of url. // And bytes body will be unmarshalled to given v. func (a *Agent) Struct(v interface{}, resp ...*Response) (code int, body []byte, errs []error) { + if a.jsonDecoder == nil { + a.jsonDecoder = json.Unmarshal + } + code, body, errs = a.Bytes(resp...) - if err := json.Unmarshal(body, v); err != nil { + if err := a.jsonDecoder(body, v); err != nil { errs = append(errs, err) } diff --git a/client_test.go b/client_test.go index 424d69bc..3dd7b6f6 100644 --- a/client_test.go +++ b/client_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/gofiber/fiber/v2/internal/encoding/json" + "github.com/gofiber/fiber/v2/utils" "github.com/valyala/fasthttp/fasthttputil" ) @@ -532,6 +534,7 @@ func Test_Client_Agent_Json(t *testing.T) { func Test_Client_Agent_Json_Error(t *testing.T) { a := Get("http://example.com"). + JSONEncoder(json.Marshal). JSON(complex(1, 1)) _, body, errs := a.String() @@ -947,7 +950,8 @@ func Test_Client_Agent_Struct(t *testing.T) { var d data - code, body, errs := a.Struct(&d) + code, body, errs := a.JSONDecoder(json.Unmarshal). + Struct(&d) utils.AssertEqual(t, StatusOK, code) utils.AssertEqual(t, `{"success"`, string(body)) diff --git a/utils/json.go b/utils/json.go new file mode 100644 index 00000000..477c8c33 --- /dev/null +++ b/utils/json.go @@ -0,0 +1,9 @@ +package utils + +// JSONMarshal returns the JSON encoding of v. +type JSONMarshal func(v interface{}) ([]byte, error) + +// JSONUnmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. +type JSONUnmarshal func(data []byte, v interface{}) error diff --git a/utils/json_marshal.go b/utils/json_marshal.go deleted file mode 100644 index 692b49a4..00000000 --- a/utils/json_marshal.go +++ /dev/null @@ -1,5 +0,0 @@ -package utils - -// JSONMarshal is the standard definition of representing a Go structure in -// json format -type JSONMarshal func(interface{}) ([]byte, error) diff --git a/utils/json_marshal_test.go b/utils/json_marshal_test.go deleted file mode 100644 index 08501d96..00000000 --- a/utils/json_marshal_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package utils - -import ( - "encoding/json" - "testing" -) - -func TestDefaultJSONEncoder(t *testing.T) { - type SampleStructure struct { - ImportantString string `json:"important_string"` - } - - var ( - sampleStructure = &SampleStructure{ - ImportantString: "Hello World", - } - importantString = `{"important_string":"Hello World"}` - - jsonEncoder JSONMarshal = json.Marshal - ) - - raw, err := jsonEncoder(sampleStructure) - AssertEqual(t, err, nil) - - AssertEqual(t, string(raw), importantString) -} diff --git a/utils/json_test.go b/utils/json_test.go new file mode 100644 index 00000000..966faa83 --- /dev/null +++ b/utils/json_test.go @@ -0,0 +1,41 @@ +package utils + +import ( + "encoding/json" + "testing" +) + +type sampleStructure struct { + ImportantString string `json:"important_string"` +} + +func Test_DefaultJSONEncoder(t *testing.T) { + t.Parallel() + + var ( + ss = &sampleStructure{ + ImportantString: "Hello World", + } + importantString = `{"important_string":"Hello World"}` + jsonEncoder JSONMarshal = json.Marshal + ) + + raw, err := jsonEncoder(ss) + AssertEqual(t, err, nil) + + AssertEqual(t, string(raw), importantString) +} + +func Test_DefaultJSONDecoder(t *testing.T) { + t.Parallel() + + var ( + ss sampleStructure + importantString = []byte(`{"important_string":"Hello World"}`) + jsonDecoder JSONUnmarshal = json.Unmarshal + ) + + err := jsonDecoder(importantString, &ss) + AssertEqual(t, err, nil) + AssertEqual(t, "Hello World", ss.ImportantString) +} From a2eab0d754e0563827fc55a92b89138cac0fd11b Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 08:19:44 +0800 Subject: [PATCH 15/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 62 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/client.go b/client.go index 3db4cb6f..82631f25 100644 --- a/client.go +++ b/client.go @@ -142,23 +142,29 @@ func (c *Client) createAgent(method, url string) *Agent { // Agent is an object storing all request data for client. type Agent struct { - *fasthttp.HostClient - req *Request - customReq *Request - args *Args - timeout time.Duration - errs []error - formFiles []*FormFile - debugWriter io.Writer - mw multipartWriter - jsonEncoder utils.JSONMarshal - jsonDecoder utils.JSONUnmarshal - maxRedirectsCount int - boundary string - Name string + // Name is used in User-Agent request header. + Name string + // NoDefaultUserAgentHeader when set to true, causes the default + // User-Agent header to be excluded from the Request. NoDefaultUserAgentHeader bool - reuse bool - parsed bool + + // HostClient is an embedded fasthttp HostClient + *fasthttp.HostClient + + req *Request + customReq *Request + args *Args + timeout time.Duration + errs []error + formFiles []*FormFile + debugWriter io.Writer + mw multipartWriter + jsonEncoder utils.JSONMarshal + jsonDecoder utils.JSONUnmarshal + maxRedirectsCount int + boundary string + reuse bool + parsed bool } // Parse initializes URI and HostClient. @@ -498,8 +504,8 @@ func (a *Agent) XML(v interface{}) *Agent { // Form sends request with body if args is non-nil. // -// It is recommended obtaining args via AcquireArgs -// in performance-critical code. +// It is recommended obtaining args via AcquireArgs and release it +// manually in performance-critical code. func (a *Agent) Form(args *Args) *Agent { a.req.Header.SetContentType(MIMEApplicationForm) @@ -525,8 +531,8 @@ type FormFile struct { // FileData appends files for multipart form request. // -// It is recommended obtaining formFile via AcquireFormFile -// in performance-critical code. +// It is recommended obtaining formFile via AcquireFormFile and release it +// manually in performance-critical code. func (a *Agent) FileData(formFiles ...*FormFile) *Agent { a.formFiles = append(a.formFiles, formFiles...) @@ -582,8 +588,8 @@ func (a *Agent) Boundary(boundary string) *Agent { // MultipartForm sends multipart form request with k-v and files. // -// It is recommended obtaining args via AcquireArgs -// in performance-critical code. +// It is recommended obtaining args via AcquireArgs and release it +// manually in performance-critical code. func (a *Agent) MultipartForm(args *Args) *Agent { if a.mw == nil { a.mw = multipart.NewWriter(a.req.BodyWriter()) @@ -646,6 +652,9 @@ func (a *Agent) Timeout(timeout time.Duration) *Agent { } // Reuse indicates the createAgent can be used again after one request. +// +// If agent is reusable, then it should be released manually when it is no +// longer used. func (a *Agent) Reuse() *Agent { a.reuse = true @@ -697,6 +706,9 @@ func (a *Agent) JSONDecoder(jsonDecoder utils.JSONUnmarshal) *Agent { /************************** End Agent Setting **************************/ // Bytes returns the status code, bytes body and errors of url. +// +// It is recommended obtaining custom response via AcquireResponse and release it +// manually in performance-critical code. func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []error) { defer a.release() @@ -765,6 +777,9 @@ func printDebugInfo(req *Request, resp *Response, w io.Writer) { } // String returns the status code, string body and errors of url. +// +// It is recommended obtaining custom response via AcquireResponse and release it +// manually in performance-critical code. func (a *Agent) String(resp ...*Response) (int, string, []error) { code, body, errs := a.Bytes(resp...) @@ -773,6 +788,9 @@ func (a *Agent) String(resp ...*Response) (int, string, []error) { // Struct returns the status code, bytes body and errors of url. // And bytes body will be unmarshalled to given v. +// +// It is recommended obtaining custom response via AcquireResponse and release it +// manually in performance-critical code. func (a *Agent) Struct(v interface{}, resp ...*Response) (code int, body []byte, errs []error) { if a.jsonDecoder == nil { a.jsonDecoder = json.Unmarshal From bc9651d58bafdabb1a4e49c1b2259325b2541c39 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 08:38:54 +0800 Subject: [PATCH 16/24] =?UTF-8?q?=F0=9F=91=B7=20Remove=20custom=20request?= =?UTF-8?q?=20and=20export=20agent=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 51 ++++++++------------------------------------------ client_test.go | 6 ++---- 2 files changed, 10 insertions(+), 47 deletions(-) diff --git a/client.go b/client.go index 82631f25..5e0c7fec 100644 --- a/client.go +++ b/client.go @@ -141,6 +141,7 @@ func (c *Client) createAgent(method, url string) *Agent { } // Agent is an object storing all request data for client. +// Agent instance MUST NOT be used from concurrently running goroutines. type Agent struct { // Name is used in User-Agent request header. Name string @@ -152,7 +153,6 @@ type Agent struct { *fasthttp.HostClient req *Request - customReq *Request args *Args timeout time.Duration errs []error @@ -174,12 +174,7 @@ func (a *Agent) Parse() error { } a.parsed = true - req := a.req - if a.customReq != nil { - req = a.customReq - } - - uri := req.URI() + uri := a.req.URI() isTLS := false scheme := uri.Scheme() @@ -465,13 +460,6 @@ func (a *Agent) BodyStream(bodyStream io.Reader, bodySize int) *Agent { return a } -// Request sets custom request for createAgent. -func (a *Agent) Request(req *Request) *Agent { - a.customReq = req - - return a -} - // JSON sends a JSON request. func (a *Agent) JSON(v interface{}) *Agent { if a.jsonEncoder == nil { @@ -703,6 +691,11 @@ func (a *Agent) JSONDecoder(jsonDecoder utils.JSONUnmarshal) *Agent { return a } +// Request returns Agent request instance. +func (a *Agent) Request() *Request { + return a.req +} + /************************** End Agent Setting **************************/ // Bytes returns the status code, bytes body and errors of url. @@ -717,9 +710,6 @@ func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []er } req := a.req - if a.customReq != nil { - req = a.customReq - } var ( resp *Response @@ -816,7 +806,6 @@ func (a *Agent) release() { func (a *Agent) reset() { a.HostClient = nil a.req.Reset() - a.customReq = nil a.timeout = 0 a.args = nil a.errs = a.errs[:0] @@ -878,7 +867,7 @@ func ReleaseClient(c *Client) { func AcquireAgent() *Agent { v := agentPool.Get() if v == nil { - return &Agent{req: fasthttp.AcquireRequest()} + return &Agent{req: &Request{}} } return v.(*Agent) } @@ -892,30 +881,6 @@ func ReleaseAgent(a *Agent) { agentPool.Put(a) } -// AcquireRequest returns an empty Request instance from request pool. -// -// The returned Request instance may be passed to ReleaseRequest when it is -// no longer needed. This allows Request recycling, reduces GC pressure -// and usually improves performance. -// Copy from fasthttp -func AcquireRequest() *Request { - v := requestPool.Get() - if v == nil { - return &Request{} - } - return v.(*Request) -} - -// ReleaseRequest returns req acquired via AcquireRequest to request pool. -// -// It is forbidden accessing req and/or its' members after returning -// it to request pool. -// Copy from fasthttp -func ReleaseRequest(req *Request) { - req.Reset() - requestPool.Put(req) -} - // AcquireResponse returns an empty Response instance from response pool. // // The returned Response instance may be passed to ReleaseResponse when it is diff --git a/client_test.go b/client_test.go index 3dd7b6f6..566f53e9 100644 --- a/client_test.go +++ b/client_test.go @@ -480,7 +480,7 @@ func Test_Client_Agent_BodyStream(t *testing.T) { testAgent(t, handler, wrapAgent, "body stream") } -func Test_Client_Agent_Custom_Request_And_Response(t *testing.T) { +func Test_Client_Agent_Custom_Response(t *testing.T) { t.Parallel() ln := fasthttputil.NewInmemoryListener() @@ -495,12 +495,11 @@ func Test_Client_Agent_Custom_Request_And_Response(t *testing.T) { for i := 0; i < 5; i++ { a := AcquireAgent() - req := AcquireRequest() resp := AcquireResponse() + req := a.Request() req.Header.SetMethod(MethodGet) req.SetRequestURI("http://example.com") - a.Request(req) utils.AssertEqual(t, nil, a.Parse()) @@ -513,7 +512,6 @@ func Test_Client_Agent_Custom_Request_And_Response(t *testing.T) { utils.AssertEqual(t, "custom", string(resp.Body())) utils.AssertEqual(t, 0, len(errs)) - ReleaseRequest(req) ReleaseResponse(resp) } } From 169001c2e1befeee8bf157d5edb3e33bd3b12768 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 08:47:19 +0800 Subject: [PATCH 17/24] =?UTF-8?q?=F0=9F=91=B7=20fix=20golint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client_test.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/client_test.go b/client_test.go index 566f53e9..ff525227 100644 --- a/client_test.go +++ b/client_test.go @@ -32,7 +32,7 @@ func Test_Client_Invalid_URL(t *testing.T) { return c.SendString(c.Hostname()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() a := Get("http://example.com\r\n\r\nGET /\r\n\r\n") @@ -69,7 +69,7 @@ func Test_Client_Get(t *testing.T) { return c.SendString(c.Hostname()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { a := Get("http://example.com") @@ -95,7 +95,7 @@ func Test_Client_Head(t *testing.T) { return c.SendString(c.Hostname()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { a := Head("http://example.com") @@ -122,7 +122,7 @@ func Test_Client_Post(t *testing.T) { SendString(c.FormValue("foo")) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { args := AcquireArgs() @@ -155,7 +155,7 @@ func Test_Client_Put(t *testing.T) { return c.SendString(c.FormValue("foo")) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { args := AcquireArgs() @@ -188,7 +188,7 @@ func Test_Client_Patch(t *testing.T) { return c.SendString(c.FormValue("foo")) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { args := AcquireArgs() @@ -222,7 +222,7 @@ func Test_Client_Delete(t *testing.T) { SendString("deleted") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { args := AcquireArgs() @@ -252,7 +252,7 @@ func Test_Client_UserAgent(t *testing.T) { return c.Send(c.Request().Header.UserAgent()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() t.Run("default", func(t *testing.T) { for i := 0; i < 5; i++ { @@ -395,7 +395,7 @@ func Test_Client_Agent_Host(t *testing.T) { return c.SendString(c.Hostname()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() a := Get("http://1.1.1.1:8080"). Host("example.com"). @@ -491,7 +491,7 @@ func Test_Client_Agent_Custom_Response(t *testing.T) { return c.SendString("custom") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { a := AcquireAgent() @@ -604,7 +604,7 @@ func Test_Client_Agent_MultipartForm(t *testing.T) { return c.Send(c.Request().Body()) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() args := AcquireArgs() @@ -674,7 +674,7 @@ func Test_Client_Agent_MultipartForm_SendFiles(t *testing.T) { return c.SendString("multipart form files") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() for i := 0; i < 5; i++ { ff := AcquireFormFile() @@ -783,7 +783,7 @@ func Test_Client_Agent_Timeout(t *testing.T) { return c.SendString("timeout") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() a := Get("http://example.com"). Timeout(time.Millisecond * 100) @@ -808,7 +808,7 @@ func Test_Client_Agent_Reuse(t *testing.T) { return c.SendString("reuse") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() a := Get("http://example.com"). Reuse() @@ -850,7 +850,7 @@ func Test_Client_Agent_TLS(t *testing.T) { return c.SendString("tls") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() code, body, errs := Get("https://" + ln.Addr().String()). InsecureSkipVerify(). @@ -880,7 +880,7 @@ func Test_Client_Agent_MaxRedirectsCount(t *testing.T) { return c.SendString("redirect") }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() t.Run("success", func(t *testing.T) { a := Get("http://example.com?foo"). @@ -924,7 +924,7 @@ func Test_Client_Agent_Struct(t *testing.T) { return c.SendString(`{"success"`) }) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() t.Run("success", func(t *testing.T) { a := Get("http://example.com") @@ -980,7 +980,7 @@ func testAgent(t *testing.T, handler Handler, wrapAgent func(agent *Agent), exce app.Get("/", handler) - go app.Listener(ln) //nolint:errcheck + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() c := 1 if len(count) > 0 { From b5402e0f3830a6eaf7e71fbeb30c4e3929939ac9 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 09:13:24 +0800 Subject: [PATCH 18/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20SetResponse=20to=20s?= =?UTF-8?q?et=20custom=20response=20for=20full=20control=20of=20response?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 46 ++++++++++++++++++++++++---------------------- client_test.go | 3 ++- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/client.go b/client.go index 5e0c7fec..02de7b5d 100644 --- a/client.go +++ b/client.go @@ -153,6 +153,7 @@ type Agent struct { *fasthttp.HostClient req *Request + resp *Response args *Args timeout time.Duration errs []error @@ -696,13 +697,20 @@ func (a *Agent) Request() *Request { return a.req } -/************************** End Agent Setting **************************/ - -// Bytes returns the status code, bytes body and errors of url. +// SetResponse sets custom response for the Agent instance. // // It is recommended obtaining custom response via AcquireResponse and release it // manually in performance-critical code. -func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []error) { +func (a *Agent) SetResponse(customResp *Response) *Agent { + a.resp = customResp + + return a +} + +/************************** End Agent Setting **************************/ + +// Bytes returns the status code, bytes body and errors of url. +func (a *Agent) Bytes() (code int, body []byte, errs []error) { defer a.release() if errs = append(errs, a.errs...); len(errs) > 0 { @@ -712,14 +720,14 @@ func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []er req := a.req var ( - resp *Response - releaseResp bool + resp *Response + nilResp bool ) - if len(customResp) > 0 { - resp = customResp[0] - } else { + if a.resp == nil { resp = AcquireResponse() - releaseResp = true + nilResp = true + } else { + resp = a.resp } defer func() { if a.debugWriter != nil { @@ -730,7 +738,7 @@ func (a *Agent) Bytes(customResp ...*Response) (code int, body []byte, errs []er code = resp.StatusCode() } - if releaseResp { + if nilResp { body = append(body, resp.Body()...) ReleaseResponse(resp) } else { @@ -767,26 +775,20 @@ func printDebugInfo(req *Request, resp *Response, w io.Writer) { } // String returns the status code, string body and errors of url. -// -// It is recommended obtaining custom response via AcquireResponse and release it -// manually in performance-critical code. -func (a *Agent) String(resp ...*Response) (int, string, []error) { - code, body, errs := a.Bytes(resp...) +func (a *Agent) String() (int, string, []error) { + code, body, errs := a.Bytes() return code, getString(body), errs } // Struct returns the status code, bytes body and errors of url. // And bytes body will be unmarshalled to given v. -// -// It is recommended obtaining custom response via AcquireResponse and release it -// manually in performance-critical code. -func (a *Agent) Struct(v interface{}, resp ...*Response) (code int, body []byte, errs []error) { +func (a *Agent) Struct(v interface{}) (code int, body []byte, errs []error) { if a.jsonDecoder == nil { a.jsonDecoder = json.Unmarshal } - code, body, errs = a.Bytes(resp...) + code, body, errs = a.Bytes() if err := a.jsonDecoder(body, v); err != nil { errs = append(errs, err) @@ -806,6 +808,7 @@ func (a *Agent) release() { func (a *Agent) reset() { a.HostClient = nil a.req.Reset() + a.resp = nil a.timeout = 0 a.args = nil a.errs = a.errs[:0] @@ -829,7 +832,6 @@ func (a *Agent) reset() { var ( clientPool sync.Pool agentPool sync.Pool - requestPool sync.Pool responsePool sync.Pool argsPool sync.Pool formFilePool sync.Pool diff --git a/client_test.go b/client_test.go index ff525227..2f1fa306 100644 --- a/client_test.go +++ b/client_test.go @@ -505,7 +505,8 @@ func Test_Client_Agent_Custom_Response(t *testing.T) { a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - code, body, errs := a.String(resp) + code, body, errs := a.SetResponse(resp). + String() utils.AssertEqual(t, StatusOK, code) utils.AssertEqual(t, "custom", body) From 1fddaed0725cc3b5e14d44cd4d955ab2c9847994 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 09:47:47 +0800 Subject: [PATCH 19/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20Dest=20for=20reusing?= =?UTF-8?q?=20returned=20body=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 17 ++++++++++++++--- client_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 02de7b5d..b8e546ca 100644 --- a/client.go +++ b/client.go @@ -154,6 +154,7 @@ type Agent struct { req *Request resp *Response + dest []byte args *Args timeout time.Duration errs []error @@ -707,6 +708,16 @@ func (a *Agent) SetResponse(customResp *Response) *Agent { return a } +// Dest sets custom dest. +// +// The contents of dest will be replaced by the response body, if the dest +// is too small a new slice will be allocated. +func (a *Agent) Dest(dest []byte) *Agent { + a.dest = dest + + return a +} + /************************** End Agent Setting **************************/ // Bytes returns the status code, bytes body and errors of url. @@ -738,11 +749,10 @@ func (a *Agent) Bytes() (code int, body []byte, errs []error) { code = resp.StatusCode() } + body = append(a.dest[:0], resp.Body()...) + if nilResp { - body = append(body, resp.Body()...) ReleaseResponse(resp) - } else { - body = resp.Body() } }() @@ -809,6 +819,7 @@ func (a *Agent) reset() { a.HostClient = nil a.req.Reset() a.resp = nil + a.dest = nil a.timeout = 0 a.args = nil a.errs = a.errs[:0] diff --git a/client_test.go b/client_test.go index 2f1fa306..539b7377 100644 --- a/client_test.go +++ b/client_test.go @@ -517,6 +517,50 @@ func Test_Client_Agent_Custom_Response(t *testing.T) { } } +func Test_Client_Agent_Dest(t *testing.T) { + t.Parallel() + + ln := fasthttputil.NewInmemoryListener() + + app := New(Config{DisableStartupMessage: true}) + + app.Get("/", func(c *Ctx) error { + return c.SendString("dest") + }) + + go func() { utils.AssertEqual(t, nil, app.Listener(ln)) }() + + t.Run("small dest", func(t *testing.T) { + dest := []byte("de") + + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.Dest(dest).String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "dest", body) + utils.AssertEqual(t, "de", string(dest)) + utils.AssertEqual(t, 0, len(errs)) + }) + + t.Run("enough dest", func(t *testing.T) { + dest := []byte("foobar") + + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + code, body, errs := a.Dest(dest).String() + + utils.AssertEqual(t, StatusOK, code) + utils.AssertEqual(t, "dest", body) + utils.AssertEqual(t, "destar", string(dest)) + utils.AssertEqual(t, 0, len(errs)) + }) +} + func Test_Client_Agent_Json(t *testing.T) { handler := func(c *Ctx) error { utils.AssertEqual(t, MIMEApplicationJSON, string(c.Request().Header.ContentType())) From 2772af030e63aade0a1eb04e4b33beacd07d5804 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 10:06:38 +0800 Subject: [PATCH 20/24] =?UTF-8?q?=F0=9F=91=B7=20handle=20pre=20errors=20fo?= =?UTF-8?q?r=20Struct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 9 ++++++--- client_test.go | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index b8e546ca..0d4669c7 100644 --- a/client.go +++ b/client.go @@ -728,18 +728,19 @@ func (a *Agent) Bytes() (code int, body []byte, errs []error) { return } - req := a.req - var ( + req = a.req resp *Response nilResp bool ) + if a.resp == nil { resp = AcquireResponse() nilResp = true } else { resp = a.resp } + defer func() { if a.debugWriter != nil { printDebugInfo(req, resp, a.debugWriter) @@ -798,7 +799,9 @@ func (a *Agent) Struct(v interface{}) (code int, body []byte, errs []error) { a.jsonDecoder = json.Unmarshal } - code, body, errs = a.Bytes() + if code, body, errs = a.Bytes(); len(errs) > 0 { + return + } if err := a.jsonDecoder(body, v); err != nil { errs = append(errs, err) diff --git a/client_test.go b/client_test.go index 539b7377..ee1830fc 100644 --- a/client_test.go +++ b/client_test.go @@ -986,6 +986,21 @@ func Test_Client_Agent_Struct(t *testing.T) { utils.AssertEqual(t, true, d.Success) }) + t.Run("pre error", func(t *testing.T) { + a := Get("http://example.com") + + a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + + var d data + + _, body, errs := a.Timeout(time.Nanosecond).Struct(&d) + + utils.AssertEqual(t, "", string(body)) + utils.AssertEqual(t, 1, len(errs)) + utils.AssertEqual(t, "timeout", errs[0].Error()) + utils.AssertEqual(t, false, d.Success) + }) + t.Run("error", func(t *testing.T) { a := Get("http://example.com/error") @@ -993,8 +1008,7 @@ func Test_Client_Agent_Struct(t *testing.T) { var d data - code, body, errs := a.JSONDecoder(json.Unmarshal). - Struct(&d) + code, body, errs := a.JSONDecoder(json.Unmarshal).Struct(&d) utils.AssertEqual(t, StatusOK, code) utils.AssertEqual(t, `{"success"`, string(body)) From f9d1074635e84902e4a96ca8e8016bd8c49f329d Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 14:37:31 +0800 Subject: [PATCH 21/24] =?UTF-8?q?=F0=9F=91=B7=20Should=20not=20force=20Age?= =?UTF-8?q?nt.dest=20to=20Agent.dest[:0]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 2 +- client_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 0d4669c7..b76a77ab 100644 --- a/client.go +++ b/client.go @@ -750,7 +750,7 @@ func (a *Agent) Bytes() (code int, body []byte, errs []error) { code = resp.StatusCode() } - body = append(a.dest[:0], resp.Body()...) + body = append(a.dest, resp.Body()...) if nilResp { ReleaseResponse(resp) diff --git a/client_test.go b/client_test.go index ee1830fc..73fc94e6 100644 --- a/client_test.go +++ b/client_test.go @@ -537,7 +537,7 @@ func Test_Client_Agent_Dest(t *testing.T) { a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - code, body, errs := a.Dest(dest).String() + code, body, errs := a.Dest(dest[:0]).String() utils.AssertEqual(t, StatusOK, code) utils.AssertEqual(t, "dest", body) @@ -552,7 +552,7 @@ func Test_Client_Agent_Dest(t *testing.T) { a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } - code, body, errs := a.Dest(dest).String() + code, body, errs := a.Dest(dest[:0]).String() utils.AssertEqual(t, StatusOK, code) utils.AssertEqual(t, "dest", body) From 9bcfb5109d3f1afe9a9679c033f36588af4afb99 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Mon, 22 Feb 2021 14:48:00 +0800 Subject: [PATCH 22/24] =?UTF-8?q?=F0=9F=91=B7=20Improve=20test=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client_test.go b/client_test.go index 73fc94e6..10aa97e3 100644 --- a/client_test.go +++ b/client_test.go @@ -990,14 +990,14 @@ func Test_Client_Agent_Struct(t *testing.T) { a := Get("http://example.com") a.HostClient.Dial = func(addr string) (net.Conn, error) { return ln.Dial() } + a.errs = append(a.errs, errors.New("pre errors")) var d data - - _, body, errs := a.Timeout(time.Nanosecond).Struct(&d) + _, body, errs := a.Struct(&d) utils.AssertEqual(t, "", string(body)) utils.AssertEqual(t, 1, len(errs)) - utils.AssertEqual(t, "timeout", errs[0].Error()) + utils.AssertEqual(t, "pre errors", errs[0].Error()) utils.AssertEqual(t, false, d.Success) }) From 09ff81716c010aa697dcce3a4990ba3371e715e9 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Thu, 25 Feb 2021 10:41:39 +0800 Subject: [PATCH 23/24] =?UTF-8?q?=F0=9F=91=B7=20Add=20beta=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client.go b/client.go index b76a77ab..ef7bec75 100644 --- a/client.go +++ b/client.go @@ -722,6 +722,8 @@ func (a *Agent) Dest(dest []byte) *Agent { // Bytes returns the status code, bytes body and errors of url. func (a *Agent) Bytes() (code int, body []byte, errs []error) { + fmt.Println("[Warning] client is still in beta, API might change in the future!") + defer a.release() if errs = append(errs, a.errs...); len(errs) > 0 { From 6ea71e9464b54e4514b07b43c8fa00d3e5615513 Mon Sep 17 00:00:00 2001 From: Kiyon Date: Fri, 26 Feb 2021 10:48:53 +0800 Subject: [PATCH 24/24] =?UTF-8?q?=20=F0=9F=8D=BA=20Cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/client.go b/client.go index ef7bec75..b5c36943 100644 --- a/client.go +++ b/client.go @@ -145,6 +145,7 @@ func (c *Client) createAgent(method, url string) *Agent { type Agent struct { // Name is used in User-Agent request header. Name string + // NoDefaultUserAgentHeader when set to true, causes the default // User-Agent header to be excluded from the Request. NoDefaultUserAgentHeader bool @@ -390,7 +391,7 @@ func (a *Agent) Host(host string) *Agent { return a } -// HostBytes sets host for the uri. +// HostBytes sets host for the URI. func (a *Agent) HostBytes(host []byte) *Agent { a.req.URI().SetHostBytes(host) @@ -492,7 +493,7 @@ func (a *Agent) XML(v interface{}) *Agent { return a } -// Form sends request with body if args is non-nil. +// Form sends form request with body if args is non-nil. // // It is recommended obtaining args via AcquireArgs and release it // manually in performance-critical code. @@ -641,7 +642,7 @@ func (a *Agent) Timeout(timeout time.Duration) *Agent { return a } -// Reuse indicates the createAgent can be used again after one request. +// Reuse enables the Agent instance to be used again after one request. // // If agent is reusable, then it should be released manually when it is no // longer used. @@ -651,7 +652,7 @@ func (a *Agent) Reuse() *Agent { return a } -// InsecureSkipVerify controls whether the createAgent verifies the server's +// InsecureSkipVerify controls whether the Agent verifies the server // certificate chain and host name. func (a *Agent) InsecureSkipVerify() *Agent { if a.HostClient.TLSConfig == nil { @@ -877,7 +878,7 @@ func ReleaseClient(c *Client) { clientPool.Put(c) } -// AcquireAgent returns an empty Agent instance from createAgent pool. +// AcquireAgent returns an empty Agent instance from Agent pool. // // The returned Agent instance may be passed to ReleaseAgent when it is // no longer needed. This allows Agent recycling, reduces GC pressure @@ -890,10 +891,10 @@ func AcquireAgent() *Agent { return v.(*Agent) } -// ReleaseAgent returns a acquired via AcquireAgent to createAgent pool. +// ReleaseAgent returns a acquired via AcquireAgent to Agent pool. // // It is forbidden accessing req and/or its' members after returning -// it to createAgent pool. +// it to Agent pool. func ReleaseAgent(a *Agent) { a.reset() agentPool.Put(a)