GOMODULES
/
pgx
mirror of https://github.com/jackc/pgx.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve stdlib performance with large results

pull/783/head
Jack Christensen 2020-06-06 09:49:14 -05:00
parent 73d0ac206c
commit 81140f6c27
4 changed files with 147 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -8,8 +8,9 @@ require (
github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853
github.com/jackc/pgio v1.0.0
github.com/jackc/pgproto3/v2 v2.0.1
github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c
github.com/jackc/puddle v1.1.1
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/rs/zerolog v1.15.0
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc
github.com/sirupsen/logrus v1.4.2

10
go.sum
View File

@ -60,6 +60,8 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a h1:XUNeoL8E15IgWouQ
github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c h1:dkoQjaMKaLf/zTPpbgbZjnU1qN4bpDY+uLBSyiKh/ns=
github.com/jackc/pgtype v1.3.1-0.20200513130519-238967ec4e4c/go.mod h1:f3c+S645fwV5ZqwPvLWZmmnAfPkmaTeLnXs0byan+aA=
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c h1:nXT9KGu1TBy57S0XcCGkTUgqOrvj3jY7Yb+kw5Q2HVc=
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
@ -95,12 +97,16 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -172,11 +178,15 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

2
rows.go
View File

@ -206,7 +206,7 @@ func (rows *connRows) Scan(dest ...interface{}) error {
if dst == nil {
continue
}
rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, values[i], dest[i])
rows.scanPlans[i] = ci.PlanScan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, dest[i])
}
}

240
stdlib/sql.go
View File

@ -404,15 +404,14 @@ func (s *Stmt) QueryContext(ctx context.Context, argsV []driver.NamedValue) (dri
return s.conn.QueryContext(ctx, s.sd.Name, argsV)
}
type rowValueFunc func(src []byte) (driver.Value, error)
type Rows struct {
conn *Conn
rows pgx.Rows
values []interface{}
driverValuers []driver.Valuer
textDecoders []pgtype.TextDecoder
binaryDecoders []pgtype.BinaryDecoder
skipNext bool
skipNextMore bool
conn *Conn
rows pgx.Rows
valueFuncs []rowValueFunc
skipNext bool
skipNextMore bool
}
func (r *Rows) Columns() []string {
@ -504,109 +503,147 @@ func (r *Rows) Next(dest []driver.Value) error {
ci := r.conn.conn.ConnInfo()
fieldDescriptions := r.rows.FieldDescriptions()
if r.values == nil {
r.values = make([]interface{}, len(fieldDescriptions))
r.driverValuers = make([]driver.Valuer, len(fieldDescriptions))
r.textDecoders = make([]pgtype.TextDecoder, len(fieldDescriptions))
r.binaryDecoders = make([]pgtype.BinaryDecoder, len(fieldDescriptions))
if r.valueFuncs == nil {
r.valueFuncs = make([]rowValueFunc, len(fieldDescriptions))
for i, fd := range fieldDescriptions {
switch fd.DataTypeOID {
case pgtype.BoolOID:
v := &pgtype.Bool{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d bool
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.ByteaOID:
v := &pgtype.Bytea{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d []byte
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.CIDOID:
v := &pgtype.CID{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.CID
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.DateOID:
v := &pgtype.Date{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.Date
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.Float4OID:
v := &pgtype.Float4{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d float32
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.Float8OID:
v := &pgtype.Float8{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d float64
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.Int2OID:
v := &pgtype.Int2{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d int16
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.Int4OID:
v := &pgtype.Int4{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d int32
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.Int8OID:
v := &pgtype.Int8{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d int64
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
case pgtype.JSONOID:
v := &pgtype.JSON{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.JSON
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.JSONBOID:
v := &pgtype.JSONB{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.JSONB
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.OIDOID:
v := &pgtype.OIDValue{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.OIDValue
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.TimestampOID:
v := &pgtype.Timestamp{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.Timestamp
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.TimestamptzOID:
v := &pgtype.Timestamptz{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.Timestamptz
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.XIDOID:
v := &pgtype.XID{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
r.binaryDecoders[i] = v
var d pgtype.XID
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
default:
v := &pgtype.GenericText{}
r.values[i] = v
r.driverValuers[i] = v
r.textDecoders[i] = v
var d string
scanPlan := ci.PlanScan(fd.DataTypeOID, fd.Format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(ci, fd.DataTypeOID, fd.Format, src, &d)
return d, err
}
}
}
}
@ -628,23 +665,14 @@ func (r *Rows) Next(dest []driver.Value) error {
}
for i, rv := range r.rows.RawValues() {
fd := fieldDescriptions[i]
if fd.Format == pgx.BinaryFormatCode {
err := r.binaryDecoders[i].DecodeBinary(ci, rv)
if rv != nil {
var err error
dest[i], err = r.valueFuncs[i](rv)
if err != nil {
return fmt.Errorf("scan field %d failed: %v", i, err)
return fmt.Errorf("convert field %d failed: %v", i, err)
}
} else {
err := r.textDecoders[i].DecodeText(ci, rv)
if err != nil {
return fmt.Errorf("scan field %d failed: %v", i, err)
}
}
var err error
dest[i], err = r.driverValuers[i].Value()
if err != nil {
return fmt.Errorf("convert field %d failed: %v", i, err)
dest[i] = nil
}
}