From 81140f6c27c97a9410046e444355d7f97b55a159 Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 6 Jun 2020 09:49:14 -0500 Subject: [PATCH] Improve stdlib performance with large results --- go.mod | 3 +- go.sum | 10 +++ rows.go | 2 +- stdlib/sql.go | 240 ++++++++++++++++++++++++++++---------------------- 4 files changed, 147 insertions(+), 108 deletions(-) diff --git a/go.mod b/go.mod index a390a967..1441d374 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,9 @@ require ( github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853 github.com/jackc/pgio v1.0.0 github.com/jackc/pgproto3/v2 v2.0.1 - github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c + github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c github.com/jackc/puddle v1.1.1 + github.com/mattn/go-colorable v0.1.6 // indirect github.com/rs/zerolog v1.15.0 github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 6da235de..8ce0705f 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a h1:XUNeoL8E15IgWouQ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c h1:dkoQjaMKaLf/zTPpbgbZjnU1qN4bpDY+uLBSyiKh/ns= github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c/go.mod h1:f3c+S645fwV5ZqwPvLWZmmnAfPkmaTeLnXs0byan+aA= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c h1:nXT9KGu1TBy57S0XcCGkTUgqOrvj3jY7Yb+kw5Q2HVc= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -95,12 +97,16 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -172,11 +178,15 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= diff --git a/rows.go b/rows.go index 612a81bc..adb6c878 100644 --- a/rows.go +++ b/rows.go @@ -206,7 +206,7 @@ func (rows *connRows) Scan(dest ...interface{}) error { if dst == nil { continue } - rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, values[i], dest[i]) + rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, dest[i]) } } diff --git a/stdlib/sql.go b/stdlib/sql.go index 76dd4578..a1ebea6c 100644 --- a/stdlib/sql.go +++ b/stdlib/sql.go @@ -404,15 +404,14 @@ func (s *Stmt) QueryContext(ctx context.Context, argsV []driver.NamedValue) (dri return s.conn.QueryContext(ctx, s.sd.Name, argsV) } +type rowValueFunc func(src []byte) (driver.Value, error) + type Rows struct { - conn *Conn - rows pgx.Rows - values []interface{} - driverValuers []driver.Valuer - textDecoders []pgtype.TextDecoder - binaryDecoders []pgtype.BinaryDecoder - skipNext bool - skipNextMore bool + conn *Conn + rows pgx.Rows + valueFuncs []rowValueFunc + skipNext bool + skipNextMore bool } func (r *Rows) Columns() []string { @@ -504,109 +503,147 @@ func (r *Rows) Next(dest []driver.Value) error { ci := r.conn.conn.ConnInfo() fieldDescriptions := r.rows.FieldDescriptions() - if r.values == nil { - r.values = make([]interface{}, len(fieldDescriptions)) - r.driverValuers = make([]driver.Valuer, len(fieldDescriptions)) - r.textDecoders = make([]pgtype.TextDecoder, len(fieldDescriptions)) - r.binaryDecoders = make([]pgtype.BinaryDecoder, len(fieldDescriptions)) + if r.valueFuncs == nil { + r.valueFuncs = make([]rowValueFunc, len(fieldDescriptions)) for i, fd := range fieldDescriptions { switch fd.DataTypeOID { case pgtype.BoolOID: - v := &pgtype.Bool{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d bool + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.ByteaOID: - v := &pgtype.Bytea{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d []byte + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.CIDOID: - v := &pgtype.CID{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.CID + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.DateOID: - v := &pgtype.Date{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.Date + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.Float4OID: - v := &pgtype.Float4{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d float32 + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.Float8OID: - v := &pgtype.Float8{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d float64 + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.Int2OID: - v := &pgtype.Int2{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d int16 + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.Int4OID: - v := &pgtype.Int4{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d int32 + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.Int8OID: - v := &pgtype.Int8{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d int64 + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } case pgtype.JSONOID: - v := &pgtype.JSON{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.JSON + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.JSONBOID: - v := &pgtype.JSONB{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.JSONB + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.OIDOID: - v := &pgtype.OIDValue{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.OIDValue + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.TimestampOID: - v := &pgtype.Timestamp{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.Timestamp + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.TimestamptzOID: - v := &pgtype.Timestamptz{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.Timestamptz + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } case pgtype.XIDOID: - v := &pgtype.XID{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v - r.binaryDecoders[i] = v + var d pgtype.XID + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + if err != nil { + return nil, err + } + return d.Value() + } default: - v := &pgtype.GenericText{} - r.values[i] = v - r.driverValuers[i] = v - r.textDecoders[i] = v + var d string + scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d) + return d, err + } } } } @@ -628,23 +665,14 @@ func (r *Rows) Next(dest []driver.Value) error { } for i, rv := range r.rows.RawValues() { - fd := fieldDescriptions[i] - if fd.Format == pgx.BinaryFormatCode { - err := r.binaryDecoders[i].DecodeBinary(ci, rv) + if rv != nil { + var err error + dest[i], err = r.valueFuncs[i](rv) if err != nil { - return fmt.Errorf("scan field %d failed: %v", i, err) + return fmt.Errorf("convert field %d failed: %v", i, err) } } else { - err := r.textDecoders[i].DecodeText(ci, rv) - if err != nil { - return fmt.Errorf("scan field %d failed: %v", i, err) - } - } - - var err error - dest[i], err = r.driverValuers[i].Value() - if err != nil { - return fmt.Errorf("convert field %d failed: %v", i, err) + dest[i] = nil } }