From 6c1be9b2201014cf1be4ab1edbee8d7ff193952c Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 13 Apr 2019 18:55:02 -0500 Subject: [PATCH] Add pool configuration MaxConns is only knob at moment --- go.mod | 9 ++++----- go.sum | 23 ++++++++++------------- pool/pool.go | 40 +++++++++++++++++++++++++++++++++++++--- pool/pool_test.go | 23 +++++++++++++++-------- pool/stat.go | 10 +++++----- 5 files changed, 71 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 7dd92c2e..6555eb6e 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,12 @@ go 1.12 require ( github.com/cockroachdb/apd v1.1.0 - github.com/go-stack/stack v1.8.0 // indirect github.com/jackc/pgconn v0.0.0-20190405170659-7ad3625edd3b github.com/jackc/pgio v1.0.0 github.com/jackc/pgproto3 v1.0.0 - github.com/jackc/puddle v0.0.0-20190409004018-0d93e0ec116a + github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b + github.com/kr/pretty v0.1.0 // indirect github.com/lib/pq v1.0.0 - github.com/mattn/go-colorable v0.1.1 // indirect - github.com/mattn/go-isatty v0.0.7 // indirect github.com/pkg/errors v0.8.1 github.com/rs/zerolog v1.13.0 github.com/satori/go.uuid v1.2.0 @@ -21,5 +19,6 @@ require ( go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 - gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec + golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 3dda696b..d52b7e00 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/pgconn v0.0.0-20190330221323-ed7d91dc9873 h1:M68R77AKFS7dub7R7WgJ9D6yiNuExOYhBuGtazlbr10= -github.com/jackc/pgconn v0.0.0-20190330221323-ed7d91dc9873/go.mod h1:8Bzf8vzi/ZpcgLgrq8IUHjZX4ZU+Hf6N6/AJ85+fDeE= github.com/jackc/pgconn v0.0.0-20190405170659-7ad3625edd3b h1:EFVP6CIXsShWT+mEeoVGMvJz1a1SAKqw8zQ15bz7cw8= github.com/jackc/pgconn v0.0.0-20190405170659-7ad3625edd3b/go.mod h1:8Bzf8vzi/ZpcgLgrq8IUHjZX4ZU+Hf6N6/AJ85+fDeE= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= @@ -18,16 +14,17 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.0.0 h1:25tUmlES7eyD96oYaUHc1dLOFbgcJtFzCdnOOoqmA1I= github.com/jackc/pgproto3 v1.0.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/puddle v0.0.0-20190409004018-0d93e0ec116a h1:zx0j45Wa4oRefVk0D3muLxUujnMWN7ZRraF+78DXEwE= -github.com/jackc/puddle v0.0.0-20190409004018-0d93e0ec116a/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b h1:cIcUpcEP55F/QuZWEtXyqHoWk+IV4TBiLjtBkeq/Q1c= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -55,5 +52,5 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTu golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pool/pool.go b/pool/pool.go index 3be32774..5bfa8e08 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -2,6 +2,8 @@ package pool import ( "context" + "fmt" + "strconv" "time" "github.com/jackc/pgconn" @@ -9,24 +11,36 @@ import ( "github.com/jackc/puddle" ) +const defaultMaxConns = 5 + type Pool struct { p *puddle.Pool } +type Config struct { + MaxConns int32 + ConnConfig *pgx.ConnConfig +} + // Connect creates a new Pool and immediately establishes one connection. ctx can be used to cancel this initial // connection. func Connect(ctx context.Context, connString string) (*Pool, error) { + config, err := ParseConfig(connString) + if err != nil { + return nil, err + } + p := &Pool{} - maxConnections := 5 // TODO - unhard-code p.p = puddle.NewPool( - func(ctx context.Context) (interface{}, error) { return pgx.Connect(ctx, connString) }, + func(ctx context.Context) (interface{}, error) { return pgx.ConnectConfig(ctx, config.ConnConfig) }, func(value interface{}) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) value.(*pgx.Conn).Close(ctx) cancel() }, - maxConnections) + config.MaxConns, + ) // Initially establish one connection res, err := p.p.Acquire(ctx) @@ -39,6 +53,26 @@ func Connect(ctx context.Context, connString string) (*Pool, error) { return p, nil } +func ParseConfig(connString string) (*Config, error) { + connConfig, err := pgx.ParseConfig(connString) + if err != nil { + return nil, err + } + + config := &Config{ConnConfig: connConfig, MaxConns: defaultMaxConns} + + if s, ok := config.ConnConfig.Config.RuntimeParams["pool_max_conns"]; ok { + delete(connConfig.Config.RuntimeParams, "pool_max_conns") + n, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid pool_max_conns: %v", err) + } + config.MaxConns = int32(n) + } + + return config, nil +} + // Close closes all connections in the pool and rejects future Acquire calls. Blocks until all connections are returned // to pool and closed. func (p *Pool) Close() { diff --git a/pool/pool_test.go b/pool/pool_test.go index f5e71c84..12533d12 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -17,6 +17,13 @@ func TestConnect(t *testing.T) { pool.Close() } +func TestParseConfigExtractsPoolArguments(t *testing.T) { + config, err := pool.ParseConfig("pool_max_conns=42") + assert.NoError(t, err) + assert.EqualValues(t, 42, config.MaxConns) + assert.NotContains(t, config.ConnConfig.Config.RuntimeParams, "pool_max_conns") +} + func TestConnectCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -57,16 +64,16 @@ func TestPoolQuery(t *testing.T) { require.NoError(t, err) stats := pool.Stat() - assert.Equal(t, 1, stats.AcquiredConns()) - assert.Equal(t, 1, stats.TotalConns()) + assert.EqualValues(t, 1, stats.AcquiredConns()) + assert.EqualValues(t, 1, stats.TotalConns()) rows.Close() assert.NoError(t, rows.Err()) waitForReleaseToComplete() stats = pool.Stat() - assert.Equal(t, 0, stats.AcquiredConns()) - assert.Equal(t, 1, stats.TotalConns()) + assert.EqualValues(t, 0, stats.AcquiredConns()) + assert.EqualValues(t, 1, stats.TotalConns()) } @@ -79,8 +86,8 @@ func TestPoolQueryRow(t *testing.T) { waitForReleaseToComplete() stats := pool.Stat() - assert.Equal(t, 0, stats.AcquiredConns()) - assert.Equal(t, 1, stats.TotalConns()) + assert.EqualValues(t, 0, stats.AcquiredConns()) + assert.EqualValues(t, 1, stats.TotalConns()) } func TestConnReleaseRollsBackFailedTransaction(t *testing.T) { @@ -162,12 +169,12 @@ func TestConnReleaseDestroysClosedConn(t *testing.T) { c.Conn().Close(ctx) - assert.Equal(t, 1, pool.Stat().TotalConns()) + assert.EqualValues(t, 1, pool.Stat().TotalConns()) c.Release() waitForReleaseToComplete() - assert.Equal(t, 0, pool.Stat().TotalConns()) + assert.EqualValues(t, 0, pool.Stat().TotalConns()) } func TestConnPoolQueryConcurrentLoad(t *testing.T) { diff --git a/pool/stat.go b/pool/stat.go index 186eefd5..a08ec7d0 100644 --- a/pool/stat.go +++ b/pool/stat.go @@ -18,7 +18,7 @@ func (s *Stat) AcquireDuration() time.Duration { return s.s.AcquireDuration() } -func (s *Stat) AcquiredConns() int { +func (s *Stat) AcquiredConns() int32 { return s.s.AcquiredResources() } @@ -26,7 +26,7 @@ func (s *Stat) CanceledAcquireCount() int64 { return s.s.CanceledAcquireCount() } -func (s *Stat) ConstructingConns() int { +func (s *Stat) ConstructingConns() int32 { return s.s.ConstructingResources() } @@ -34,14 +34,14 @@ func (s *Stat) EmptyAcquireCount() int64 { return s.s.EmptyAcquireCount() } -func (s *Stat) IdleConns() int { +func (s *Stat) IdleConns() int32 { return s.s.IdleResources() } -func (s *Stat) MaxConns() int { +func (s *Stat) MaxConns() int32 { return s.s.MaxResources() } -func (s *Stat) TotalConns() int { +func (s *Stat) TotalConns() int32 { return s.s.TotalResources() }