Improve stdlib performance with large results

go.mod

@@ -8,8 +8,9 @@ require (
 	github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853
 	github.com/jackc/pgio v1.0.0
 	github.com/jackc/pgproto3/v2 v2.0.1
-	github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c
+	github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c
 	github.com/jackc/puddle v1.1.1
+	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/rs/zerolog v1.15.0
 	github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc
 	github.com/sirupsen/logrus v1.4.2

go.sum

@@ -60,6 +60,8 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a h1:XUNeoL8E15IgWouQ
 github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
 github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c h1:dkoQjaMKaLf/zTPpbgbZjnU1qN4bpDY+uLBSyiKh/ns=
 github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c/go.mod h1:f3c+S645fwV5ZqwPvLWZmmnAfPkmaTeLnXs0byan+aA=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c h1:nXT9KGu1TBy57S0XcCGkTUgqOrvj3jY7Yb+kw5Q2HVc=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
 github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
@@ -95,12 +97,16 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -172,11 +178,15 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

rows.go

@@ -206,7 +206,7 @@ func (rows *connRows) Scan(dest ...interface{}) error {
 			if dst == nil {
 				continue
 			}
-			rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, values[i], dest[i])
+			rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, dest[i])
 		}
 	}
 

stdlib/sql.go

@@ -404,15 +404,14 @@ func (s *Stmt) QueryContext(ctx context.Context, argsV []driver.NamedValue) (dri
 	return s.conn.QueryContext(ctx, s.sd.Name, argsV)
 }
 
+type rowValueFunc func(src []byte) (driver.Value, error)
+
 type Rows struct {
-	conn           *Conn
+	conn         *Conn
-	rows           pgx.Rows
+	rows         pgx.Rows
-	values         []interface{}
+	valueFuncs   []rowValueFunc
-	driverValuers  []driver.Valuer
+	skipNext     bool
-	textDecoders   []pgtype.TextDecoder
+	skipNextMore bool
-	binaryDecoders []pgtype.BinaryDecoder
-	skipNext       bool
-	skipNextMore   bool
 }
 
 func (r *Rows) Columns() []string {
@@ -504,109 +503,147 @@ func (r *Rows) Next(dest []driver.Value) error {
 	ci := r.conn.conn.ConnInfo()
 	fieldDescriptions := r.rows.FieldDescriptions()
 
-	if r.values == nil {
+	if r.valueFuncs == nil {
-		r.values = make([]interface{}, len(fieldDescriptions))
+		r.valueFuncs = make([]rowValueFunc, len(fieldDescriptions))
-		r.driverValuers = make([]driver.Valuer, len(fieldDescriptions))
-		r.textDecoders = make([]pgtype.TextDecoder, len(fieldDescriptions))
-		r.binaryDecoders = make([]pgtype.BinaryDecoder, len(fieldDescriptions))
 
 		for i, fd := range fieldDescriptions {
 			switch fd.DataTypeOID {
 			case pgtype.BoolOID:
-				v := &pgtype.Bool{}
+				var d bool
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.ByteaOID:
-				v := &pgtype.Bytea{}
+				var d []byte
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.CIDOID:
-				v := &pgtype.CID{}
+				var d pgtype.CID
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.DateOID:
-				v := &pgtype.Date{}
+				var d pgtype.Date
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.Float4OID:
-				v := &pgtype.Float4{}
+				var d float32
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.Float8OID:
-				v := &pgtype.Float8{}
+				var d float64
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.Int2OID:
-				v := &pgtype.Int2{}
+				var d int16
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.Int4OID:
-				v := &pgtype.Int4{}
+				var d int32
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.Int8OID:
-				v := &pgtype.Int8{}
+				var d int64
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					return d, err
+				}
 			case pgtype.JSONOID:
-				v := &pgtype.JSON{}
+				var d pgtype.JSON
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.JSONBOID:
-				v := &pgtype.JSONB{}
+				var d pgtype.JSONB
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.OIDOID:
-				v := &pgtype.OIDValue{}
+				var d pgtype.OIDValue
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.TimestampOID:
-				v := &pgtype.Timestamp{}
+				var d pgtype.Timestamp
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.TimestamptzOID:
-				v := &pgtype.Timestamptz{}
+				var d pgtype.Timestamptz
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			case pgtype.XIDOID:
-				v := &pgtype.XID{}
+				var d pgtype.XID
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
-				r.binaryDecoders[i] = v
+					if err != nil {
+						return nil, err
+					}
+					return d.Value()
+				}
 			default:
-				v := &pgtype.GenericText{}
+				var d string
-				r.values[i] = v
+				scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
-				r.driverValuers[i] = v
+				r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
-				r.textDecoders[i] = v
+					err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
+					return d, err
+				}
 			}
 		}
 	}
@@ -628,23 +665,14 @@ func (r *Rows) Next(dest []driver.Value) error {
 	}
 
 	for i, rv := range r.rows.RawValues() {
-		fd := fieldDescriptions[i]
+		if rv != nil {
-		if fd.Format == pgx.BinaryFormatCode {
+			var err error
-			err := r.binaryDecoders[i].DecodeBinary(ci, rv)
+			dest[i], err = r.valueFuncs[i](rv)
 			if err != nil {
-				return fmt.Errorf("scan field %d failed: %v", i, err)
+				return fmt.Errorf("convert field %d failed: %v", i, err)
 			}
 		} else {
-			err := r.textDecoders[i].DecodeText(ci, rv)
+			dest[i] = nil
-			if err != nil {
-				return fmt.Errorf("scan field %d failed: %v", i, err)
-			}
-		}
-
-		var err error
-		dest[i], err = r.driverValuers[i].Value()
-		if err != nil {
-			return fmt.Errorf("convert field %d failed: %v", i, err)
 		}
 	}