From edecbf8191c88e70729e389b36e467c963198501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Garcia?= Date: Sun, 16 May 2021 17:05:21 -0300 Subject: [PATCH] Add feature of omiting the "SELECT" part of the query Now the 3 functions that allow you to write plain SQL queries also work if you omit the `SELECT ...` part of the query. If you do this the code will check and notice that the first token of the query is a "FROM" token and then automatically build the SELECT part of the query based on the tags of the struct. Everything is cached, so the impact on performance should be negligible. The affected functions are: - Query() - QueryOne() - QueryChunks() --- go.mod | 4 +- ksql.go | 69 +++ ksql_test.go | 1172 +++++++++++++++++++++++--------------------- structs/structs.go | 4 + 4 files changed, 687 insertions(+), 562 deletions(-) diff --git a/go.mod b/go.mod index f09a85a..3dac923 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/vingarcia/ksql go 1.14 require ( - github.com/denisenkom/go-mssqldb v0.10.0 // indirect + github.com/denisenkom/go-mssqldb v0.10.0 github.com/ditointernet/go-assert v0.0.0-20200120164340-9e13125a7018 - github.com/go-sql-driver/mysql v1.4.0 // indirect + github.com/go-sql-driver/mysql v1.4.0 github.com/golang/mock v1.5.0 github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.1.1 diff --git a/ksql.go b/ksql.go index 86ba670..1c30a8b 100644 --- a/ksql.go +++ b/ksql.go @@ -6,11 +6,20 @@ import ( "fmt" "reflect" "strings" + "unicode" "github.com/pkg/errors" "github.com/vingarcia/ksql/structs" ) +var selectQueryCache = map[string]map[reflect.Type]string{} + +func init() { + for dname := range supportedDialects { + selectQueryCache[dname] = map[reflect.Type]string{} + } +} + // DB represents the ksql client responsible for // interfacing with the "database/sql" package implementing // the KissSQL interface `SQLProvider`. @@ -124,6 +133,14 @@ func (c DB) Query( slice = slice.Slice(0, 0) } + if strings.ToUpper(getFirstToken(query)) == "FROM" { + selectPrefix, err := buildSelectQuery(c.dialect, structType, selectQueryCache[c.dialect.DriverName()]) + if err != nil { + return err + } + query = selectPrefix + query + } + rows, err := c.db.QueryContext(ctx, query, params...) if err != nil { return fmt.Errorf("error running query: %s", err.Error()) @@ -189,6 +206,14 @@ func (c DB) QueryOne( return fmt.Errorf("ksql: expected to receive a pointer to struct, but got: %T", record) } + if strings.ToUpper(getFirstToken(query)) == "FROM" { + selectPrefix, err := buildSelectQuery(c.dialect, t, selectQueryCache[c.dialect.DriverName()]) + if err != nil { + return err + } + query = selectPrefix + query + } + rows, err := c.db.QueryContext(ctx, query, params...) if err != nil { return err @@ -243,6 +268,14 @@ func (c DB) QueryChunks( return err } + if strings.ToUpper(getFirstToken(parser.Query)) == "FROM" { + selectPrefix, err := buildSelectQuery(c.dialect, structType, selectQueryCache[c.dialect.DriverName()]) + if err != nil { + return err + } + parser.Query = selectPrefix + parser.Query + } + rows, err := c.db.QueryContext(ctx, parser.Query, parser.Params...) if err != nil { return err @@ -869,3 +902,39 @@ func buildCompositeKeyDeleteQuery( strings.Join(values, ","), ), params } + +// We implemented this function instead of using +// a regex or strings.Fields because we wanted +// to preserve the performance of the package. +func getFirstToken(s string) string { + s = strings.TrimLeftFunc(s, unicode.IsSpace) + + var token strings.Builder + for _, c := range s { + if unicode.IsSpace(c) { + break + } + token.WriteRune(c) + } + return token.String() +} + +func buildSelectQuery( + dialect dialect, + structType reflect.Type, + selectQueryCache map[reflect.Type]string, +) (string, error) { + if selectQuery, found := selectQueryCache[structType]; found { + return selectQuery, nil + } + + info := structs.GetTagInfo(structType) + var fields []string + for _, field := range info.Fields() { + fields = append(fields, dialect.Escape(field.Name)) + } + + query := "SELECT " + strings.Join(fields, ", ") + " " + selectQueryCache[structType] = query + return query, nil +} diff --git a/ksql_test.go b/ksql_test.go index b8132d5..19a6cad 100644 --- a/ksql_test.go +++ b/ksql_test.go @@ -37,144 +37,162 @@ type Address struct { func TestQuery(t *testing.T) { for driver := range supportedDialects { t.Run(driver, func(t *testing.T) { - t.Run("using slice of structs", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } + variations := []struct { + desc string + queryPrefix string + }{ + { + desc: "with select *", + queryPrefix: "SELECT * ", + }, + { + desc: "building the SELECT part of the query internally", + queryPrefix: "", + }, + } + for _, variation := range variations { + t.Run(variation.desc, func(t *testing.T) { + t.Run("using slice of structs", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } - t.Run("should return 0 results correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() + t.Run("should return 0 results correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []User - err := c.Query(ctx, &users, `SELECT * FROM users WHERE id=1;`) - assert.Equal(t, nil, err) - assert.Equal(t, []User(nil), users) + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []User + err := c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE id=1;`) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(users)) - users = []User{} - err = c.Query(ctx, &users, `SELECT * FROM users WHERE id=1;`) - assert.Equal(t, nil, err) - assert.Equal(t, []User{}, users) + users = []User{} + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE id=1;`) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(users)) + }) + + t.Run("should return a user correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) + + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []User + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") + + assert.Equal(t, nil, err) + assert.Equal(t, 1, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "Bia", users[0].Name) + assert.Equal(t, "BR", users[0].Address.Country) + }) + + t.Run("should return multiple users correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('João Garcia', 0, '{"country":"US"}')`) + assert.Equal(t, nil, err) + + _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia Garcia', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) + + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []User + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE name like `+c.dialect.Placeholder(0), "% Garcia") + + assert.Equal(t, nil, err) + assert.Equal(t, 2, len(users)) + + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "João Garcia", users[0].Name) + assert.Equal(t, "US", users[0].Address.Country) + + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "Bia Garcia", users[1].Name) + assert.Equal(t, "BR", users[1].Address.Country) + }) + }) + + t.Run("using slice of pointers to structs", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + t.Run("should return 0 results correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []*User + err := c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE id=1;`) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(users)) + + users = []*User{} + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE id=1;`) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(users)) + }) + + t.Run("should return a user correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) + + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []*User + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") + + assert.Equal(t, nil, err) + assert.Equal(t, 1, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "Bia", users[0].Name) + assert.Equal(t, "BR", users[0].Address.Country) + }) + + t.Run("should return multiple users correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('João Garcia', 0, '{"country":"US"}')`) + assert.Equal(t, nil, err) + + _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia Garcia', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) + + ctx := context.Background() + c := newTestDB(db, driver, "users") + var users []*User + err = c.Query(ctx, &users, variation.queryPrefix+`FROM users WHERE name like `+c.dialect.Placeholder(0), "% Garcia") + + assert.Equal(t, nil, err) + assert.Equal(t, 2, len(users)) + + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "João Garcia", users[0].Name) + assert.Equal(t, "US", users[0].Address.Country) + + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "Bia Garcia", users[1].Name) + assert.Equal(t, "BR", users[1].Address.Country) + }) + }) }) - - t.Run("should return a user correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) - - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []User - err = c.Query(ctx, &users, `SELECT * FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") - - assert.Equal(t, nil, err) - assert.Equal(t, 1, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "Bia", users[0].Name) - assert.Equal(t, "BR", users[0].Address.Country) - }) - - t.Run("should return multiple users correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('João Garcia', 0, '{"country":"US"}')`) - assert.Equal(t, nil, err) - - _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia Garcia', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) - - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []User - err = c.Query(ctx, &users, `SELECT * FROM users WHERE name like `+c.dialect.Placeholder(0), "% Garcia") - - assert.Equal(t, nil, err) - assert.Equal(t, 2, len(users)) - - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "João Garcia", users[0].Name) - assert.Equal(t, "US", users[0].Address.Country) - - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "Bia Garcia", users[1].Name) - assert.Equal(t, "BR", users[1].Address.Country) - }) - }) - - t.Run("using slice of pointers to structs", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - t.Run("should return 0 results correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []*User - err := c.Query(ctx, &users, `SELECT * FROM users WHERE id=1;`) - assert.Equal(t, nil, err) - assert.Equal(t, []*User(nil), users) - - users = []*User{} - err = c.Query(ctx, &users, `SELECT * FROM users WHERE id=1;`) - assert.Equal(t, nil, err) - assert.Equal(t, []*User{}, users) - }) - - t.Run("should return a user correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) - - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []*User - err = c.Query(ctx, &users, `SELECT * FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") - - assert.Equal(t, nil, err) - assert.Equal(t, 1, len(users)) - assert.Equal(t, "Bia", users[0].Name) - assert.NotEqual(t, uint(0), users[0].ID) - }) - - t.Run("should return multiple users correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('João Garcia', 0, '{"country":"US"}')`) - assert.Equal(t, nil, err) - - _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia Garcia', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) - - ctx := context.Background() - c := newTestDB(db, driver, "users") - var users []*User - err = c.Query(ctx, &users, `SELECT * FROM users WHERE name like `+c.dialect.Placeholder(0), "% Garcia") - - assert.Equal(t, nil, err) - assert.Equal(t, 2, len(users)) - - assert.Equal(t, "João Garcia", users[0].Name) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "US", users[0].Address.Country) - - assert.Equal(t, "Bia Garcia", users[1].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "BR", users[1].Address.Country) - }) - }) + } t.Run("testing error cases", func(t *testing.T) { err := createTable(driver) @@ -226,64 +244,81 @@ func TestQuery(t *testing.T) { func TestQueryOne(t *testing.T) { for driver := range supportedDialects { t.Run(driver, func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) + variations := []struct { + desc string + queryPrefix string + }{ + { + desc: "with select *", + queryPrefix: "SELECT * ", + }, + { + desc: "building the SELECT part of the query internally", + queryPrefix: "", + }, } + for _, variation := range variations { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } - t.Run("should return RecordNotFoundErr when there are no results", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() + t.Run(variation.desc, func(t *testing.T) { + t.Run("should return RecordNotFoundErr when there are no results", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() - ctx := context.Background() - c := newTestDB(db, driver, "users") - u := User{} - err := c.QueryOne(ctx, &u, `SELECT * FROM users WHERE id=1;`) - assert.Equal(t, ErrRecordNotFound, err) - }) + ctx := context.Background() + c := newTestDB(db, driver, "users") + u := User{} + err := c.QueryOne(ctx, &u, variation.queryPrefix+`FROM users WHERE id=1;`) + assert.Equal(t, ErrRecordNotFound, err) + }) - t.Run("should return a user correctly", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() + t.Run("should return a user correctly", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Bia', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) - ctx := context.Background() - c := newTestDB(db, driver, "users") - u := User{} - err = c.QueryOne(ctx, &u, `SELECT * FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") + ctx := context.Background() + c := newTestDB(db, driver, "users") + u := User{} + err = c.QueryOne(ctx, &u, variation.queryPrefix+`FROM users WHERE name=`+c.dialect.Placeholder(0), "Bia") - assert.Equal(t, nil, err) - assert.NotEqual(t, uint(0), u.ID) - assert.Equal(t, "Bia", u.Name) - assert.Equal(t, Address{ - Country: "BR", - }, u.Address) - }) + assert.Equal(t, nil, err) + assert.NotEqual(t, uint(0), u.ID) + assert.Equal(t, "Bia", u.Name) + assert.Equal(t, Address{ + Country: "BR", + }, u.Address) + }) - t.Run("should return only the first result on multiples matches", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() + t.Run("should return only the first result on multiples matches", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() - _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Andréa Sá', 0, '{"country":"US"}')`) - assert.Equal(t, nil, err) + _, err := db.Exec(`INSERT INTO users (name, age, address) VALUES ('Andréa Sá', 0, '{"country":"US"}')`) + assert.Equal(t, nil, err) - _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Caio Sá', 0, '{"country":"BR"}')`) - assert.Equal(t, nil, err) + _, err = db.Exec(`INSERT INTO users (name, age, address) VALUES ('Caio Sá', 0, '{"country":"BR"}')`) + assert.Equal(t, nil, err) - ctx := context.Background() - c := newTestDB(db, driver, "users") + ctx := context.Background() + c := newTestDB(db, driver, "users") - var u User - err = c.QueryOne(ctx, &u, `SELECT * FROM users WHERE name like `+c.dialect.Placeholder(0)+` ORDER BY id ASC`, "% Sá") - assert.Equal(t, nil, err) - assert.Equal(t, "Andréa Sá", u.Name) - assert.Equal(t, 0, u.Age) - assert.Equal(t, Address{ - Country: "US", - }, u.Address) - }) + var u User + err = c.QueryOne(ctx, &u, variation.queryPrefix+`FROM users WHERE name like `+c.dialect.Placeholder(0)+` ORDER BY id ASC`, "% Sá") + assert.Equal(t, nil, err) + assert.Equal(t, "Andréa Sá", u.Name) + assert.Equal(t, 0, u.Age) + assert.Equal(t, Address{ + Country: "US", + }, u.Address) + }) + }) + } t.Run("should report error if input is not a pointer to struct", func(t *testing.T) { db := connectDB(t, driver) @@ -312,7 +347,7 @@ func TestQueryOne(t *testing.T) { ctx := context.Background() c := newTestDB(db, driver, "users") var user User - err = c.QueryOne(ctx, &user, `SELECT * FROM not a valid query`) + err := c.QueryOne(ctx, &user, `SELECT * FROM not a valid query`) assert.NotEqual(t, nil, err) }) }) @@ -762,397 +797,414 @@ func TestUpdate(t *testing.T) { func TestQueryChunks(t *testing.T) { for driver := range supportedDialects { t.Run(driver, func(t *testing.T) { - t.Run("should query a single row correctly", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{ - Name: "User1", - Address: Address{Country: "BR"}, - }) - - var length int - var u User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `SELECT * FROM users WHERE name = ` + c.dialect.Placeholder(0), - Params: []interface{}{"User1"}, - - ChunkSize: 100, - ForEachChunk: func(users []User) error { - length = len(users) - if length > 0 { - u = users[0] + variations := []struct { + desc string + queryPrefix string + }{ + { + desc: "with select *", + queryPrefix: "SELECT * ", + }, + { + desc: "building the SELECT part of the query internally", + queryPrefix: "", + }, + } + for _, variation := range variations { + t.Run(variation.desc, func(t *testing.T) { + t.Run("should query a single row correctly", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) } - return nil - }, - }) - assert.Equal(t, nil, err) - assert.Equal(t, 1, length) - assert.NotEqual(t, uint(0), u.ID) - assert.Equal(t, "User1", u.Name) - assert.Equal(t, "BR", u.Address.Country) - }) + db := connectDB(t, driver) + defer db.Close() - t.Run("should query one chunk correctly", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } + ctx := context.Background() + c := newTestDB(db, driver, "users") - db := connectDB(t, driver) - defer db.Close() + _ = c.Insert(ctx, &User{ + Name: "User1", + Address: Address{Country: "BR"}, + }) - ctx := context.Background() - c := newTestDB(db, driver, "users") + var length int + var u User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `FROM users WHERE name = ` + c.dialect.Placeholder(0), + Params: []interface{}{"User1"}, - _ = c.Insert(ctx, &User{Name: "User1", Address: Address{Country: "US"}}) - _ = c.Insert(ctx, &User{Name: "User2", Address: Address{Country: "BR"}}) + ChunkSize: 100, + ForEachChunk: func(users []User) error { + length = len(users) + if length > 0 { + u = users[0] + } + return nil + }, + }) - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - users = append(users, buffer...) - lengths = append(lengths, len(buffer)) - return nil - }, - }) - - assert.Equal(t, nil, err) - assert.Equal(t, 1, len(lengths)) - assert.Equal(t, 2, lengths[0]) - - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.Equal(t, "US", users[0].Address.Country) - - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.Equal(t, "BR", users[1].Address.Country) - }) - - t.Run("should query chunks of 1 correctly", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1", Address: Address{Country: "US"}}) - _ = c.Insert(ctx, &User{Name: "User2", Address: Address{Country: "BR"}}) - - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 1, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - return nil - }, - }) - - assert.Equal(t, nil, err) - assert.Equal(t, 2, len(users)) - assert.Equal(t, []int{1, 1}, lengths) - - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.Equal(t, "US", users[0].Address.Country) - - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.Equal(t, "BR", users[1].Address.Country) - }) - - t.Run("should load partially filled chunks correctly", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1"}) - _ = c.Insert(ctx, &User{Name: "User2"}) - _ = c.Insert(ctx, &User{Name: "User3"}) - - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - return nil - }, - }) - - assert.Equal(t, nil, err) - assert.Equal(t, 3, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.NotEqual(t, uint(0), users[2].ID) - assert.Equal(t, "User3", users[2].Name) - assert.Equal(t, []int{2, 1}, lengths) - }) - - t.Run("should abort the first iteration when the callback returns an ErrAbortIteration", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1"}) - _ = c.Insert(ctx, &User{Name: "User2"}) - _ = c.Insert(ctx, &User{Name: "User3"}) - - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - return ErrAbortIteration - }, - }) - - assert.Equal(t, nil, err) - assert.Equal(t, 2, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.Equal(t, []int{2}, lengths) - }) - - t.Run("should abort the last iteration when the callback returns an ErrAbortIteration", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1"}) - _ = c.Insert(ctx, &User{Name: "User2"}) - _ = c.Insert(ctx, &User{Name: "User3"}) - - returnVals := []error{nil, ErrAbortIteration} - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - - return shiftErrSlice(&returnVals) - }, - }) - - assert.Equal(t, nil, err) - assert.Equal(t, 3, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.NotEqual(t, uint(0), users[2].ID) - assert.Equal(t, "User3", users[2].Name) - assert.Equal(t, []int{2, 1}, lengths) - }) - - t.Run("should return error if the callback returns an error in the first iteration", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1"}) - _ = c.Insert(ctx, &User{Name: "User2"}) - _ = c.Insert(ctx, &User{Name: "User3"}) - - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - return errors.New("fake error msg") - }, - }) - - assert.NotEqual(t, nil, err) - assert.Equal(t, 2, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.Equal(t, []int{2}, lengths) - }) - - t.Run("should return error if the callback returns an error in the last iteration", func(t *testing.T) { - err := createTable(driver) - if err != nil { - t.Fatal("could not create test table!, reason:", err.Error()) - } - - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - _ = c.Insert(ctx, &User{Name: "User1"}) - _ = c.Insert(ctx, &User{Name: "User2"}) - _ = c.Insert(ctx, &User{Name: "User3"}) - - returnVals := []error{nil, errors.New("fake error msg")} - var lengths []int - var users []User - err = c.QueryChunks(ctx, ChunkParser{ - Query: `select * from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, - Params: []interface{}{"User%"}, - - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - lengths = append(lengths, len(buffer)) - users = append(users, buffer...) - - return shiftErrSlice(&returnVals) - }, - }) - - assert.NotEqual(t, nil, err) - assert.Equal(t, 3, len(users)) - assert.NotEqual(t, uint(0), users[0].ID) - assert.Equal(t, "User1", users[0].Name) - assert.NotEqual(t, uint(0), users[1].ID) - assert.Equal(t, "User2", users[1].Name) - assert.NotEqual(t, uint(0), users[2].ID) - assert.Equal(t, "User3", users[2].Name) - assert.Equal(t, []int{2, 1}, lengths) - }) - - t.Run("should report error if the input function is invalid", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() - - ctx := context.Background() - c := newTestDB(db, driver, "users") - - funcs := []interface{}{ - nil, - "not a function", - func() error { - return nil - }, - func(extraInputValue []User, extra []User) error { - return nil - }, - func(invalidArgType string) error { - return nil - }, - func(missingReturnType []User) { - return - }, - func(users []User) string { - return "" - }, - func(extraReturnValue []User) ([]User, error) { - return nil, nil - }, - func(notSliceOfStructs []string) error { - return nil - }, - } - - for _, fn := range funcs { - err := c.QueryChunks(ctx, ChunkParser{ - Query: `SELECT * FROM users`, - Params: []interface{}{}, - - ChunkSize: 2, - ForEachChunk: fn, + assert.Equal(t, nil, err) + assert.Equal(t, 1, length) + assert.NotEqual(t, uint(0), u.ID) + assert.Equal(t, "User1", u.Name) + assert.Equal(t, "BR", u.Address.Country) }) - assert.NotEqual(t, nil, err) - } - }) - t.Run("should report error if the query is not valid", func(t *testing.T) { - db := connectDB(t, driver) - defer db.Close() + t.Run("should query one chunk correctly", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } - ctx := context.Background() - c := newTestDB(db, driver, "users") - err := c.QueryChunks(ctx, ChunkParser{ - Query: `SELECT * FROM not a valid query`, - Params: []interface{}{}, + db := connectDB(t, driver) + defer db.Close() - ChunkSize: 2, - ForEachChunk: func(buffer []User) error { - return nil - }, + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1", Address: Address{Country: "US"}}) + _ = c.Insert(ctx, &User{Name: "User2", Address: Address{Country: "BR"}}) + + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + users = append(users, buffer...) + lengths = append(lengths, len(buffer)) + return nil + }, + }) + + assert.Equal(t, nil, err) + assert.Equal(t, 1, len(lengths)) + assert.Equal(t, 2, lengths[0]) + + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.Equal(t, "US", users[0].Address.Country) + + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.Equal(t, "BR", users[1].Address.Country) + }) + + t.Run("should query chunks of 1 correctly", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1", Address: Address{Country: "US"}}) + _ = c.Insert(ctx, &User{Name: "User2", Address: Address{Country: "BR"}}) + + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 1, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + return nil + }, + }) + + assert.Equal(t, nil, err) + assert.Equal(t, 2, len(users)) + assert.Equal(t, []int{1, 1}, lengths) + + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.Equal(t, "US", users[0].Address.Country) + + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.Equal(t, "BR", users[1].Address.Country) + }) + + t.Run("should load partially filled chunks correctly", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1"}) + _ = c.Insert(ctx, &User{Name: "User2"}) + _ = c.Insert(ctx, &User{Name: "User3"}) + + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + return nil + }, + }) + + assert.Equal(t, nil, err) + assert.Equal(t, 3, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.NotEqual(t, uint(0), users[2].ID) + assert.Equal(t, "User3", users[2].Name) + assert.Equal(t, []int{2, 1}, lengths) + }) + + t.Run("should abort the first iteration when the callback returns an ErrAbortIteration", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1"}) + _ = c.Insert(ctx, &User{Name: "User2"}) + _ = c.Insert(ctx, &User{Name: "User3"}) + + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + return ErrAbortIteration + }, + }) + + assert.Equal(t, nil, err) + assert.Equal(t, 2, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.Equal(t, []int{2}, lengths) + }) + + t.Run("should abort the last iteration when the callback returns an ErrAbortIteration", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1"}) + _ = c.Insert(ctx, &User{Name: "User2"}) + _ = c.Insert(ctx, &User{Name: "User3"}) + + returnVals := []error{nil, ErrAbortIteration} + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + + return shiftErrSlice(&returnVals) + }, + }) + + assert.Equal(t, nil, err) + assert.Equal(t, 3, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.NotEqual(t, uint(0), users[2].ID) + assert.Equal(t, "User3", users[2].Name) + assert.Equal(t, []int{2, 1}, lengths) + }) + + t.Run("should return error if the callback returns an error in the first iteration", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1"}) + _ = c.Insert(ctx, &User{Name: "User2"}) + _ = c.Insert(ctx, &User{Name: "User3"}) + + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + return errors.New("fake error msg") + }, + }) + + assert.NotEqual(t, nil, err) + assert.Equal(t, 2, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.Equal(t, []int{2}, lengths) + }) + + t.Run("should return error if the callback returns an error in the last iteration", func(t *testing.T) { + err := createTable(driver) + if err != nil { + t.Fatal("could not create test table!, reason:", err.Error()) + } + + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + _ = c.Insert(ctx, &User{Name: "User1"}) + _ = c.Insert(ctx, &User{Name: "User2"}) + _ = c.Insert(ctx, &User{Name: "User3"}) + + returnVals := []error{nil, errors.New("fake error msg")} + var lengths []int + var users []User + err = c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `from users where name like ` + c.dialect.Placeholder(0) + ` order by name asc;`, + Params: []interface{}{"User%"}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + lengths = append(lengths, len(buffer)) + users = append(users, buffer...) + + return shiftErrSlice(&returnVals) + }, + }) + + assert.NotEqual(t, nil, err) + assert.Equal(t, 3, len(users)) + assert.NotEqual(t, uint(0), users[0].ID) + assert.Equal(t, "User1", users[0].Name) + assert.NotEqual(t, uint(0), users[1].ID) + assert.Equal(t, "User2", users[1].Name) + assert.NotEqual(t, uint(0), users[2].ID) + assert.Equal(t, "User3", users[2].Name) + assert.Equal(t, []int{2, 1}, lengths) + }) + + t.Run("should report error if the input function is invalid", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + + funcs := []interface{}{ + nil, + "not a function", + func() error { + return nil + }, + func(extraInputValue []User, extra []User) error { + return nil + }, + func(invalidArgType string) error { + return nil + }, + func(missingReturnType []User) { + return + }, + func(users []User) string { + return "" + }, + func(extraReturnValue []User) ([]User, error) { + return nil, nil + }, + func(notSliceOfStructs []string) error { + return nil + }, + } + + for _, fn := range funcs { + err := c.QueryChunks(ctx, ChunkParser{ + Query: variation.queryPrefix + `FROM users`, + Params: []interface{}{}, + + ChunkSize: 2, + ForEachChunk: fn, + }) + assert.NotEqual(t, nil, err) + } + }) + + t.Run("should report error if the query is not valid", func(t *testing.T) { + db := connectDB(t, driver) + defer db.Close() + + ctx := context.Background() + c := newTestDB(db, driver, "users") + err := c.QueryChunks(ctx, ChunkParser{ + Query: `SELECT * FROM not a valid query`, + Params: []interface{}{}, + + ChunkSize: 2, + ForEachChunk: func(buffer []User) error { + return nil + }, + }) + assert.NotEqual(t, nil, err) + }) }) - assert.NotEqual(t, nil, err) - }) + } }) } } diff --git a/structs/structs.go b/structs/structs.go index 7c7941d..63d7d57 100644 --- a/structs/structs.go +++ b/structs/structs.go @@ -42,6 +42,10 @@ func (s structInfo) Add(field fieldInfo) { s.byName[field.Name] = &field } +func (s structInfo) Fields() map[int]*fieldInfo { + return s.byIndex +} + // This cache is kept as a pkg variable // because the total number of types on a program // should be finite. So keeping a single cache here