From 456a242f5c8c997383522f886f171fc16039c110 Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Fri, 23 Dec 2022 13:11:28 -0600 Subject: [PATCH] Unregistered OIDs are handled the same as unknown OIDs This improves handling of unregistered types. In general, they should "just work". But there are performance benefits gained and some edge cases avoided by registering types. Updated documentation to mention this. https://github.com/jackc/pgx/issues/1445 --- doc.go | 5 +-- pgtype/doc.go | 77 +++++++++++++++++++++++++++++-------------- pgtype/pgtype.go | 12 +++---- pgtype/pgtype_test.go | 41 +++++++++++++++++++++++ query_test.go | 20 ++++++++++- 5 files changed, 121 insertions(+), 34 deletions(-) diff --git a/doc.go b/doc.go index fb271874..0db8cbb1 100644 --- a/doc.go +++ b/doc.go @@ -69,8 +69,9 @@ Use Exec to execute a query that does not return a result set. PostgreSQL Data Types -The package pgtype provides extensive and customizable support for converting Go values to and from PostgreSQL values -including array and composite types. See that package's documentation for details. +pgx uses the pgtype package to converting Go values to and from PostgreSQL values. It supports many PostgreSQL types +directly and is customizable and extendable. User defined data types such as enums, domains, and composite types may +require type registration. See that package's documentation for details. Transactions diff --git a/pgtype/doc.go b/pgtype/doc.go index 834aa0b6..6612c896 100644 --- a/pgtype/doc.go +++ b/pgtype/doc.go @@ -57,27 +57,7 @@ JSON Support pgtype automatically marshals and unmarshals data from json and jsonb PostgreSQL types. -Array Support - -ArrayCodec implements support for arrays. If pgtype supports type T then it can easily support []T by registering an -ArrayCodec for the appropriate PostgreSQL OID. In addition, Array[T] type can support multi-dimensional arrays. - -Composite Support - -CompositeCodec implements support for PostgreSQL composite types. Go structs can be scanned into if the public fields of -the struct are in the exact order and type of the PostgreSQL type or by implementing CompositeIndexScanner and -CompositeIndexGetter. - -Enum Support - -PostgreSQL enums can usually be treated as text. However, EnumCodec implements support for interning strings which can -reduce memory usage. - -Array, Composite, and Enum Type Registration - -Array, composite, and enum types can be easily registered from a pgx.Conn with the LoadType method. - -Extending Existing Type Support +Extending Existing PostgreSQL Type Support Generally, all Codecs will support interfaces that can be implemented to enable scanning and encoding. For example, PointCodec can use any Go type that implements the PointScanner and PointValuer interfaces. So rather than use @@ -90,11 +70,58 @@ pgx support such as github.com/shopspring/decimal. These types can be registered logic. See https://github.com/jackc/pgx-shopspring-decimal and https://github.com/jackc/pgx-gofrs-uuid for a example integrations. -Entirely New Type Support +New PostgreSQL Type Support -If the PostgreSQL type is not already supported then an OID / Codec mapping can be registered with Map.RegisterType. -There is no difference between a Codec defined and registered by the application and a Codec built in to pgtype. See any -of the Codecs in pgtype for Codec examples and for examples of type registration. +pgtype uses the PostgreSQL OID to determine how to encode or decode a value. pgtype supports array, composite, domain, +and enum types. However, any type created in PostgreSQL with CREATE TYPE will receive a new OID. This means that the OID +of each new PostgreSQL type must be registered for pgtype to handle values of that type with the correct Codec. + +The pgx.Conn LoadType method can return a *Type for array, composite, domain, and enum types by inspecting the database +metadata. This *Type can then be registered with Map.RegisterType. + +For example, the following function could be called after a connection is established: + + func RegisterDataTypes(ctx context.Context, conn *pgx.Conn) error { + dataTypeNames := []string{ + "foo", + "_foo", + "bar", + "_bar", + } + + for _, typeName := range dataTypeNames { + dataType, err := conn.LoadType(ctx, typeName) + if err != nil { + return err + } + conn.TypeMap().RegisterType(dataType) + } + + return nil + } + +A type cannot be registered unless all types it depends on are already registered. e.g. An array type cannot be +registered until its element type is registered. + +ArrayCodec implements support for arrays. If pgtype supports type T then it can easily support []T by registering an +ArrayCodec for the appropriate PostgreSQL OID. In addition, Array[T] type can support multi-dimensional arrays. + +CompositeCodec implements support for PostgreSQL composite types. Go structs can be scanned into if the public fields of +the struct are in the exact order and type of the PostgreSQL type or by implementing CompositeIndexScanner and +CompositeIndexGetter. + +Domain types are treated as their underlying type if the underlying type and the domain type are registered. + +PostgreSQL enums can usually be treated as text. However, EnumCodec implements support for interning strings which can +reduce memory usage. + +While pgtype will often still work with unregistered types it is highly recommended that all types be registered due to +an improvement in performance and the elimination of certain edge cases. + +If an entirely new PostgreSQL type (e.g. PostGIS types) is used then the application or a library can create a new +Codec. Then the OID / Codec mapping can be registered with Map.RegisterType. There is no difference between a Codec +defined and registered by the application and a Codec built in to pgtype. See any of the Codecs in pgtype for Codec +examples and for examples of type registration. Encoding Unknown Types diff --git a/pgtype/pgtype.go b/pgtype/pgtype.go index 7cfa3d58..ad451678 100644 --- a/pgtype/pgtype.go +++ b/pgtype/pgtype.go @@ -1326,16 +1326,16 @@ func (m *Map) PlanEncode(oid uint32, format int16, value any) EncodePlan { } var dt *Type - - if oid == 0 { + if dataType, ok := m.TypeForOID(oid); ok { + dt = dataType + } else { + // If no type for the OID was found, then either it is unknowable (e.g. the simple protocol) or it is an + // unregistered type. In either case try to find the type and OID that matches the value (e.g. a []byte would be + // registered to PostgreSQL bytea). if dataType, ok := m.TypeForValue(value); ok { dt = dataType oid = dt.OID // Preserve assumed OID in case we are recursively called below. } - } else { - if dataType, ok := m.TypeForOID(oid); ok { - dt = dataType - } } if dt != nil { diff --git a/pgtype/pgtype_test.go b/pgtype/pgtype_test.go index 9b5ec87d..c0cb3852 100644 --- a/pgtype/pgtype_test.go +++ b/pgtype/pgtype_test.go @@ -43,6 +43,10 @@ type _float32Slice []float32 type _float64Slice []float64 type _byteSlice []byte +// unregisteredOID represents a actual type that is not registered. Cannot use 0 because that represents that the type +// is not known (e.g. when using the simple protocol). +const unregisteredOID = uint32(1) + func mustParseCIDR(t testing.TB, s string) *net.IPNet { _, ipnet, err := net.ParseCIDR(s) if err != nil { @@ -127,6 +131,13 @@ func (f sqlScannerFunc) Scan(src any) error { return f(src) } +// driverValuerFunc lets an arbitrary function be used as a driver.Valuer. +type driverValuerFunc func() (driver.Value, error) + +func (f driverValuerFunc) Value() (driver.Value, error) { + return f() +} + func TestMapScanNilIsNoOp(t *testing.T) { m := pgtype.NewMap() @@ -324,6 +335,36 @@ func TestMapEncodeBinaryFormatDatabaseValuerThatReturnsString(t *testing.T) { require.Equal(t, []byte{0, 0, 0, 42}, buf) } +// https://github.com/jackc/pgx/issues/1445 +func TestMapEncodeDatabaseValuerThatReturnsStringIntoUnregisteredTypeTextFormat(t *testing.T) { + m := pgtype.NewMap() + buf, err := m.Encode(unregisteredOID, pgtype.TextFormatCode, driverValuerFunc(func() (driver.Value, error) { return "foo", nil }), nil) + require.NoError(t, err) + require.Equal(t, []byte("foo"), buf) +} + +// https://github.com/jackc/pgx/issues/1445 +func TestMapEncodeDatabaseValuerThatReturnsByteSliceIntoUnregisteredTypeTextFormat(t *testing.T) { + m := pgtype.NewMap() + buf, err := m.Encode(unregisteredOID, pgtype.TextFormatCode, driverValuerFunc(func() (driver.Value, error) { return []byte{0, 1, 2, 3}, nil }), nil) + require.NoError(t, err) + require.Equal(t, []byte(`\x00010203`), buf) +} + +func TestMapEncodeStringIntoUnregisteredTypeTextFormat(t *testing.T) { + m := pgtype.NewMap() + buf, err := m.Encode(unregisteredOID, pgtype.TextFormatCode, "foo", nil) + require.NoError(t, err) + require.Equal(t, []byte("foo"), buf) +} + +func TestMapEncodeByteSliceIntoUnregisteredTypeTextFormat(t *testing.T) { + m := pgtype.NewMap() + buf, err := m.Encode(unregisteredOID, pgtype.TextFormatCode, []byte{0, 1, 2, 3}, nil) + require.NoError(t, err) + require.Equal(t, []byte(`\x00010203`), buf) +} + // https://github.com/jackc/pgx/issues/1326 func TestMapScanPointerToRenamedType(t *testing.T) { srcBuf := []byte("foo") diff --git a/query_test.go b/query_test.go index c00cdec2..bf1a1fe8 100644 --- a/query_test.go +++ b/query_test.go @@ -190,7 +190,7 @@ func TestConnQueryValuesWhenUnableToDecode(t *testing.T) { require.Equal(t, "({1},)", values[0]) } -func TestConnQueryValuesWithUnknownOID(t *testing.T) { +func TestConnQueryValuesWithUnregisteredOID(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -217,6 +217,24 @@ func TestConnQueryValuesWithUnknownOID(t *testing.T) { require.Equal(t, "orange", values[0]) } +func TestConnQueryArgsAndScanWithUnregisteredOID(t *testing.T) { + t.Parallel() + + pgxtest.RunWithQueryExecModes(context.Background(), t, defaultConnTestRunner, nil, func(ctx context.Context, t testing.TB, conn *pgx.Conn) { + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, "create type fruit as enum('orange', 'apple', 'pear')") + require.NoError(t, err) + + var result string + err = conn.QueryRow(ctx, "select $1::fruit", "orange").Scan(&result) + require.NoError(t, err) + require.Equal(t, "orange", result) + }) +} + // https://github.com/jackc/pgx/issues/478 func TestConnQueryReadRowMultipleTimes(t *testing.T) { t.Parallel()