diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac1613e..20e63c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - go-version: [1.16, 1.17, 1.18] + go-version: [1.17, 1.18] os: [ubuntu-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a4e2beb..3b870bf 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -25,3 +25,16 @@ jobs: - name: Run e2e ${{ matrix.dialect }} tests run: | make test-e2e-${{ matrix.dialect }} + test-clickhouse: + name: Run clickhouse tests + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: "1.18" + - name: Run clickhouse test + run: make test-clickhouse diff --git a/Makefile b/Makefile index 7c98ebe..78f602b 100644 --- a/Makefile +++ b/Makefile @@ -19,5 +19,8 @@ test-e2e-postgres: test-e2e-mysql: go test -v ./tests/e2e -dialect=mysql +test-clickhouse: + go test -timeout=10m -count=1 -race -v ./tests/clickhouse -test.short + docker-cleanup: docker stop -t=0 $$(docker ps --filter="label=goose_test" -aq) diff --git a/cmd/goose/driver_clickhouse.go b/cmd/goose/driver_clickhouse.go index ecc7d56..af721b8 100644 --- a/cmd/goose/driver_clickhouse.go +++ b/cmd/goose/driver_clickhouse.go @@ -4,5 +4,5 @@ package main import ( - _ "github.com/ClickHouse/clickhouse-go" + _ "github.com/ClickHouse/clickhouse-go/v2" ) diff --git a/dialect.go b/dialect.go index 759f61b..71101ce 100644 --- a/dialect.go +++ b/dialect.go @@ -293,7 +293,7 @@ func (m TiDBDialect) deleteVersionSQL() string { type ClickHouseDialect struct{} func (m ClickHouseDialect) createVersionTableSQL() string { - return fmt.Sprintf(`CREATE TABLE %s ( + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( version_id Int64, is_applied UInt8, date Date default now(), @@ -310,7 +310,11 @@ func (m ClickHouseDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { } func (m ClickHouseDialect) insertVersionSQL() string { - return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2)", TableName()) + // Note, the second argument is_applied is hard-coded to 1 (true). This is to account for + // the /v2 ClickHouse driver not converting bool to uint8. This will hopefully be resolved + // upstream: https://github.com/ClickHouse/clickhouse-go/issues/621 + // Throughout the codebase this value is always true, hence it has no effect. + return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, 1)", TableName()) } func (m ClickHouseDialect) migrationSQL() string { diff --git a/go.mod b/go.mod index c43981b..8dc1655 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module github.com/pressly/goose/v3 -go 1.16 +go 1.17 require ( - github.com/ClickHouse/clickhouse-go v1.5.4 + github.com/ClickHouse/clickhouse-go/v2 v2.1.0 + github.com/avast/retry-go/v4 v4.1.0 github.com/denisenkom/go-mssqldb v0.12.2 github.com/go-sql-driver/mysql v1.6.0 github.com/lib/pq v1.10.6 @@ -12,3 +13,54 @@ require ( github.com/ziutek/mymysql v1.5.4 modernc.org/sqlite v1.17.3 ) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/containerd/continuity v0.3.0 // indirect + github.com/docker/cli v20.10.14+incompatible // indirect + github.com/docker/docker v20.10.7+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/runc v1.1.2 // indirect + github.com/paulmach/orb v0.7.1 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.opentelemetry.io/otel v1.7.0 // indirect + go.opentelemetry.io/otel/trace v1.7.0 // indirect + golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 // indirect + golang.org/x/tools v0.1.7 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + lukechampine.com/uint128 v1.1.1 // indirect + modernc.org/cc/v3 v3.36.0 // indirect + modernc.org/ccgo/v3 v3.16.6 // indirect + modernc.org/libc v1.16.7 // indirect + modernc.org/mathutil v1.4.1 // indirect + modernc.org/memory v1.1.1 // indirect + modernc.org/opt v0.1.1 // indirect + modernc.org/strutil v1.1.1 // indirect + modernc.org/token v1.0.0 // indirect +) diff --git a/go.sum b/go.sum index 49abde4..c2e7677 100644 --- a/go.sum +++ b/go.sum @@ -4,19 +4,21 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.1.0 h1:X53a5FzRna9TLGGYm1A7T+3kEnrfEYl15BNsL6sw81s= +github.com/ClickHouse/clickhouse-go/v2 v2.1.0/go.mod h1:nOBMOlMUGQJ2eb6PtECHYldbEHmDJFzfIrtaDXMjrb4= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/avast/retry-go/v4 v4.1.0 h1:CwudD9anYv6JMVnDuTRlK6kLo4dBamiL+F3U8YDiyfg= +github.com/avast/retry-go/v4 v4.1.0/go.mod h1:HqmLvS2VLdStPCGDFjSuZ9pzlTqVRldCI4w2dO4m1Ms= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= @@ -43,6 +45,11 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -59,12 +66,15 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= @@ -75,7 +85,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -86,6 +95,7 @@ github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -101,8 +111,13 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/ory/dockertest/v3 v3.9.1 h1:v4dkG+dlu76goxMiTT2j8zV7s4oPPEppKT8K8p2f1kY= github.com/ory/dockertest/v3 v3.9.1/go.mod h1:42Ir9hmvaAPm0Mgibk6mBPi7SFvTXxEcnztDYOJ//uM= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/paulmach/orb v0.7.1 h1:Zha++Z5OX/l168sqHK3k4z18LDvr+YAO/VjK0ReQ9rU= +github.com/paulmach/orb v0.7.1/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -111,8 +126,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= @@ -121,10 +142,14 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -136,24 +161,32 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -161,22 +194,25 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -187,15 +223,14 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -205,11 +240,11 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= -gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo= diff --git a/internal/check/check.go b/internal/check/check.go index cb097e8..177f7d0 100644 --- a/internal/check/check.go +++ b/internal/check/check.go @@ -45,6 +45,13 @@ func Number(t *testing.T, got, want interface{}) { } } +func Equal(t *testing.T, got, want interface{}) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("failed deep equal:\ngot:\t%v\nwant:\t%v\v", got, want) + } +} + func NumberNotZero(t *testing.T, got interface{}) { t.Helper() gotNumber, err := reflectToInt64(got) @@ -77,3 +84,11 @@ func reflectToInt64(v interface{}) (int64, error) { } return 0, fmt.Errorf("invalid number: must be int64 type: got:%T", v) } + +func reflectToStr(v interface{}) (string, error) { + switch typ := v.(type) { + case string: + return reflect.ValueOf(typ).String(), nil + } + return "", fmt.Errorf("invalid string: got:%T", v) +} diff --git a/internal/testdb/clickhouse.go b/internal/testdb/clickhouse.go new file mode 100644 index 0000000..1df2c17 --- /dev/null +++ b/internal/testdb/clickhouse.go @@ -0,0 +1,110 @@ +package testdb + +import ( + "crypto/tls" + "database/sql" + "fmt" + "log" + "strconv" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const ( + // https://hub.docker.com/r/clickhouse/clickhouse-server/ + CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server" + CLICKHOUSE_VERSION = "22-alpine" + + CLICKHOUSE_DB = "clickdb" + CLICKHOUSE_USER = "clickuser" + CLICKHOUSE_PASSWORD = "password1" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT = "1" +) + +func newClickHouse(opts ...OptionsFunc) (*sql.DB, func(), error) { + option := &options{} + for _, f := range opts { + f(option) + } + // Uses a sensible default on windows (tcp/http) and linux/osx (socket). + pool, err := dockertest.NewPool("") + if err != nil { + return nil, nil, err + } + runOptions := &dockertest.RunOptions{ + Repository: CLICKHOUSE_IMAGE, + Tag: CLICKHOUSE_VERSION, + Env: []string{ + "CLICKHOUSE_DB=" + CLICKHOUSE_DB, + "CLICKHOUSE_USER=" + CLICKHOUSE_USER, + "CLICKHOUSE_PASSWORD=" + CLICKHOUSE_PASSWORD, + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT, + }, + Labels: map[string]string{"goose_test": "1"}, + PortBindings: make(map[docker.Port][]docker.PortBinding), + } + // Port 8123 is used for HTTP, but we're using the TCP protocol endpoint (port 9000). + // Ref: https://clickhouse.com/docs/en/interfaces/http/ + // Ref: https://clickhouse.com/docs/en/interfaces/tcp/ + if option.bindPort > 0 { + runOptions.PortBindings[docker.Port("9000/tcp")] = []docker.PortBinding{ + {HostPort: strconv.Itoa(option.bindPort)}, + } + } + container, err := pool.RunWithOptions( + runOptions, + func(config *docker.HostConfig) { + // Set AutoRemove to true so that stopped container goes away by itself. + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }, + ) + if err != nil { + return nil, nil, err + } + cleanup := func() { + if err := pool.Purge(container); err != nil { + log.Printf("failed to purge resource: %v", err) + } + } + // Fetch port assigned to container + address := fmt.Sprintf("%s:%s", "localhost", container.GetPort("9000/tcp")) + + var db *sql.DB + // Exponential backoff-retry, because the application in the container + // might not be ready to accept connections yet. + if err := pool.Retry(func() error { + db = clickHouseOpenDB(address, nil, option.debug) + return db.Ping() + }); err != nil { + return nil, cleanup, fmt.Errorf("could not connect to docker database: %w", err) + } + return db, cleanup, nil +} + +func clickHouseOpenDB(address string, tlsConfig *tls.Config, debug bool) *sql.DB { + db := clickhouse.OpenDB(&clickhouse.Options{ + Addr: []string{address}, + Auth: clickhouse.Auth{ + Database: CLICKHOUSE_DB, + Username: CLICKHOUSE_USER, + Password: CLICKHOUSE_PASSWORD, + }, + TLS: tlsConfig, + Settings: clickhouse.Settings{ + "max_execution_time": 60, + }, + DialTimeout: 5 * time.Second, + Compression: &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }, + Debug: debug, + }) + db.SetMaxIdleConns(5) + db.SetMaxOpenConns(10) + db.SetConnMaxLifetime(time.Hour) + return db +} diff --git a/internal/testdb/options.go b/internal/testdb/options.go new file mode 100644 index 0000000..bd86812 --- /dev/null +++ b/internal/testdb/options.go @@ -0,0 +1,16 @@ +package testdb + +type options struct { + bindPort int + debug bool +} + +type OptionsFunc func(o *options) + +func WithBindPort(n int) OptionsFunc { + return func(o *options) { o.bindPort = n } +} + +func WithDebug(b bool) OptionsFunc { + return func(o *options) { o.debug = b } +} diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go new file mode 100644 index 0000000..509af12 --- /dev/null +++ b/internal/testdb/testdb.go @@ -0,0 +1,10 @@ +package testdb + +import "database/sql" + +// NewClickHouse starts a ClickHouse docker container, and returns +// a connection and a cleanup function. +// If bindPort is 0,b a random port will be used. +func NewClickHouse(options ...OptionsFunc) (_ *sql.DB, cleanup func(), _ error) { + return newClickHouse(options...) +} diff --git a/tests/clickhouse/clickhouse_test.go b/tests/clickhouse/clickhouse_test.go new file mode 100644 index 0000000..6a83b93 --- /dev/null +++ b/tests/clickhouse/clickhouse_test.go @@ -0,0 +1,209 @@ +package clickhouse_test + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/avast/retry-go/v4" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/internal/check" + "github.com/pressly/goose/v3/internal/testdb" +) + +func TestClickUpDownAll(t *testing.T) { + t.Parallel() + + migrationDir := filepath.Join("testdata", "migrations") + db, cleanup, err := testdb.NewClickHouse() + check.NoError(t, err) + t.Cleanup(cleanup) + + goose.SetDialect("clickhouse") + + retryCheckTableMutation := func(table string) func() error { + return func() error { + ok := checkTableMutation(t, db, table) + if !ok { + return errors.New("mutation not done for table: " + table) + } + return nil + } + } + + /* + This test applies all up migrations, asserts we have all the entries in + the versions table, applies all down migration and asserts we have zero + migrations applied. + + ClickHouse performs UPDATES and DELETES asynchronously, + but we can best-effort check mutations and their progress. + + This is especially important for down migrations where rows are deleted + from the versions table. + + For the sake of testing, there might be a way to modifying the server + (or queries) to perform all operations synchronously? + + Ref: https://clickhouse.com/docs/en/operations/system-tables/mutations/ + Ref: https://clickhouse.com/docs/en/sql-reference/statements/alter/#mutations + Ref: https://clickhouse.com/blog/how-to-update-data-in-click-house/ + */ + + // Collect migrations so we don't have to hard-code the currentVersion + // in an assertion later in the test. + migrations, err := goose.CollectMigrations(migrationDir, 0, goose.MaxVersion) + check.NoError(t, err) + + currentVersion, err := goose.GetDBVersion(db) + check.NoError(t, err) + check.Number(t, currentVersion, 0) + + err = goose.Up(db, migrationDir) + check.NoError(t, err) + currentVersion, err = goose.GetDBVersion(db) + check.NoError(t, err) + check.Number(t, currentVersion, len(migrations)) + + err = goose.DownTo(db, migrationDir, 0) + check.NoError(t, err) + err = retry.Do( + retryCheckTableMutation(goose.TableName()), + retry.Delay(1*time.Second), + ) + check.NoError(t, err) + + currentVersion, err = goose.GetDBVersion(db) + check.NoError(t, err) + check.Number(t, currentVersion, 0) +} + +func TestClickHouseFirstThree(t *testing.T) { + t.Parallel() + + migrationDir := filepath.Join("testdata", "migrations") + db, cleanup, err := testdb.NewClickHouse() + check.NoError(t, err) + t.Cleanup(cleanup) + + goose.SetDialect("clickhouse") + + err = goose.Up(db, migrationDir) + check.NoError(t, err) + + currentVersion, err := goose.GetDBVersion(db) + check.NoError(t, err) + check.Number(t, currentVersion, 3) + + type result struct { + customerID string `db:"customer_id"` + timestamp time.Time `db:"time_stamp"` + clickEventType string `db:"click_event_type"` + countryCode string `db:"country_code"` + sourceID int64 `db:"source_id"` + } + rows, err := db.Query(`SELECT * FROM clickstream ORDER BY customer_id`) + check.NoError(t, err) + var results []result + for rows.Next() { + var r result + err = rows.Scan(&r.customerID, &r.timestamp, &r.clickEventType, &r.countryCode, &r.sourceID) + check.NoError(t, err) + results = append(results, r) + } + check.Number(t, len(results), 3) + check.NoError(t, rows.Close()) + check.NoError(t, rows.Err()) + + parseTime := func(t *testing.T, s string) time.Time { + t.Helper() + tm, err := time.Parse("2006-01-02", s) + check.NoError(t, err) + return tm + } + want := []result{ + {"customer1", parseTime(t, "2021-10-02"), "add_to_cart", "US", 568239}, + {"customer2", parseTime(t, "2021-10-30"), "remove_from_cart", "", 0}, + {"customer3", parseTime(t, "2021-11-07"), "checkout", "", 307493}, + } + for i, result := range results { + check.Equal(t, result.customerID, want[i].customerID) + check.Equal(t, result.timestamp, want[i].timestamp) + check.Equal(t, result.clickEventType, want[i].clickEventType) + if result.countryCode != "" && want[i].countryCode != "" { + check.Equal(t, result.countryCode, want[i].countryCode) + } + check.Number(t, result.sourceID, want[i].sourceID) + } +} + +func TestRemoteImportMigration(t *testing.T) { + t.Parallel() + // TODO(mf): use TestMain and create a proper "long" or "remote" flag. + if !testing.Short() { + t.Skip("skipping test") + } + // This test is using a remote dataset from an s3 bucket: + // https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/taxi_zone_lookup.csv + // From this tutorial: https://clickhouse.com/docs/en/tutorial/ + // Note, these files are backed up in this repository in: + // tests/clickhouse/testdata/backup-files/taxi_zone_lookup.csv + // We may want to host this ourselves. Or just don't bother with SOURCE(HTTP(URL.. + // and craft a long INSERT statement. + + migrationDir := filepath.Join("testdata", "migrations-remote") + db, cleanup, err := testdb.NewClickHouse(testdb.WithBindPort(9000)) + check.NoError(t, err) + t.Cleanup(cleanup) + + goose.SetDialect("clickhouse") + + err = goose.Up(db, migrationDir) + check.NoError(t, err) + _, err = goose.GetDBVersion(db) + check.NoError(t, err) + + var count int + err = db.QueryRow(`SELECT count(*) FROM taxi_zone_dictionary`).Scan(&count) + check.NoError(t, err) + check.Number(t, count, 265) +} + +func checkTableMutation(t *testing.T, db *sql.DB, tableName string) bool { + t.Helper() + rows, err := db.Query( + `select mutation_id, command, is_done, create_time from system.mutations where table=$1`, + tableName, + ) + check.NoError(t, err) + + type result struct { + mutationID string `db:"mutation_id"` + command string `db:"command"` + isDone int64 `db:"is_done"` + createTime time.Time `db:"create_time"` + } + var results []result + for rows.Next() { + var r result + err = rows.Scan(&r.mutationID, &r.command, &r.isDone, &r.createTime) + check.NoError(t, err) + results = append(results, r) + } + check.NoError(t, rows.Close()) + check.NoError(t, rows.Err()) + // No results means there are no mutations. Assume they are all done. + if len(results) == 0 { + return true + } + // Loop through all the mutations, if at least one of them is + // not done, return false. + for _, r := range results { + if r.isDone != 1 { + return false + } + } + return true +} diff --git a/tests/clickhouse/testdata/backup-files/taxi_zone_lookup.csv b/tests/clickhouse/testdata/backup-files/taxi_zone_lookup.csv new file mode 100644 index 0000000..ecd01e0 --- /dev/null +++ b/tests/clickhouse/testdata/backup-files/taxi_zone_lookup.csv @@ -0,0 +1,266 @@ +"LocationID","Borough","Zone","service_zone" +1,"EWR","Newark Airport","EWR" +2,"Queens","Jamaica Bay","Boro Zone" +3,"Bronx","Allerton/Pelham Gardens","Boro Zone" +4,"Manhattan","Alphabet City","Yellow Zone" +5,"Staten Island","Arden Heights","Boro Zone" +6,"Staten Island","Arrochar/Fort Wadsworth","Boro Zone" +7,"Queens","Astoria","Boro Zone" +8,"Queens","Astoria Park","Boro Zone" +9,"Queens","Auburndale","Boro Zone" +10,"Queens","Baisley Park","Boro Zone" +11,"Brooklyn","Bath Beach","Boro Zone" +12,"Manhattan","Battery Park","Yellow Zone" +13,"Manhattan","Battery Park City","Yellow Zone" +14,"Brooklyn","Bay Ridge","Boro Zone" +15,"Queens","Bay Terrace/Fort Totten","Boro Zone" +16,"Queens","Bayside","Boro Zone" +17,"Brooklyn","Bedford","Boro Zone" +18,"Bronx","Bedford Park","Boro Zone" +19,"Queens","Bellerose","Boro Zone" +20,"Bronx","Belmont","Boro Zone" +21,"Brooklyn","Bensonhurst East","Boro Zone" +22,"Brooklyn","Bensonhurst West","Boro Zone" +23,"Staten Island","Bloomfield/Emerson Hill","Boro Zone" +24,"Manhattan","Bloomingdale","Yellow Zone" +25,"Brooklyn","Boerum Hill","Boro Zone" +26,"Brooklyn","Borough Park","Boro Zone" +27,"Queens","Breezy Point/Fort Tilden/Riis Beach","Boro Zone" +28,"Queens","Briarwood/Jamaica Hills","Boro Zone" +29,"Brooklyn","Brighton Beach","Boro Zone" +30,"Queens","Broad Channel","Boro Zone" +31,"Bronx","Bronx Park","Boro Zone" +32,"Bronx","Bronxdale","Boro Zone" +33,"Brooklyn","Brooklyn Heights","Boro Zone" +34,"Brooklyn","Brooklyn Navy Yard","Boro Zone" +35,"Brooklyn","Brownsville","Boro Zone" +36,"Brooklyn","Bushwick North","Boro Zone" +37,"Brooklyn","Bushwick South","Boro Zone" +38,"Queens","Cambria Heights","Boro Zone" +39,"Brooklyn","Canarsie","Boro Zone" +40,"Brooklyn","Carroll Gardens","Boro Zone" +41,"Manhattan","Central Harlem","Boro Zone" +42,"Manhattan","Central Harlem North","Boro Zone" +43,"Manhattan","Central Park","Yellow Zone" +44,"Staten Island","Charleston/Tottenville","Boro Zone" +45,"Manhattan","Chinatown","Yellow Zone" +46,"Bronx","City Island","Boro Zone" +47,"Bronx","Claremont/Bathgate","Boro Zone" +48,"Manhattan","Clinton East","Yellow Zone" +49,"Brooklyn","Clinton Hill","Boro Zone" +50,"Manhattan","Clinton West","Yellow Zone" +51,"Bronx","Co-Op City","Boro Zone" +52,"Brooklyn","Cobble Hill","Boro Zone" +53,"Queens","College Point","Boro Zone" +54,"Brooklyn","Columbia Street","Boro Zone" +55,"Brooklyn","Coney Island","Boro Zone" +56,"Queens","Corona","Boro Zone" +57,"Queens","Corona","Boro Zone" +58,"Bronx","Country Club","Boro Zone" +59,"Bronx","Crotona Park","Boro Zone" +60,"Bronx","Crotona Park East","Boro Zone" +61,"Brooklyn","Crown Heights North","Boro Zone" +62,"Brooklyn","Crown Heights South","Boro Zone" +63,"Brooklyn","Cypress Hills","Boro Zone" +64,"Queens","Douglaston","Boro Zone" +65,"Brooklyn","Downtown Brooklyn/MetroTech","Boro Zone" +66,"Brooklyn","DUMBO/Vinegar Hill","Boro Zone" +67,"Brooklyn","Dyker Heights","Boro Zone" +68,"Manhattan","East Chelsea","Yellow Zone" +69,"Bronx","East Concourse/Concourse Village","Boro Zone" +70,"Queens","East Elmhurst","Boro Zone" +71,"Brooklyn","East Flatbush/Farragut","Boro Zone" +72,"Brooklyn","East Flatbush/Remsen Village","Boro Zone" +73,"Queens","East Flushing","Boro Zone" +74,"Manhattan","East Harlem North","Boro Zone" +75,"Manhattan","East Harlem South","Boro Zone" +76,"Brooklyn","East New York","Boro Zone" +77,"Brooklyn","East New York/Pennsylvania Avenue","Boro Zone" +78,"Bronx","East Tremont","Boro Zone" +79,"Manhattan","East Village","Yellow Zone" +80,"Brooklyn","East Williamsburg","Boro Zone" +81,"Bronx","Eastchester","Boro Zone" +82,"Queens","Elmhurst","Boro Zone" +83,"Queens","Elmhurst/Maspeth","Boro Zone" +84,"Staten Island","Eltingville/Annadale/Prince's Bay","Boro Zone" +85,"Brooklyn","Erasmus","Boro Zone" +86,"Queens","Far Rockaway","Boro Zone" +87,"Manhattan","Financial District North","Yellow Zone" +88,"Manhattan","Financial District South","Yellow Zone" +89,"Brooklyn","Flatbush/Ditmas Park","Boro Zone" +90,"Manhattan","Flatiron","Yellow Zone" +91,"Brooklyn","Flatlands","Boro Zone" +92,"Queens","Flushing","Boro Zone" +93,"Queens","Flushing Meadows-Corona Park","Boro Zone" +94,"Bronx","Fordham South","Boro Zone" +95,"Queens","Forest Hills","Boro Zone" +96,"Queens","Forest Park/Highland Park","Boro Zone" +97,"Brooklyn","Fort Greene","Boro Zone" +98,"Queens","Fresh Meadows","Boro Zone" +99,"Staten Island","Freshkills Park","Boro Zone" +100,"Manhattan","Garment District","Yellow Zone" +101,"Queens","Glen Oaks","Boro Zone" +102,"Queens","Glendale","Boro Zone" +103,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +104,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +105,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +106,"Brooklyn","Gowanus","Boro Zone" +107,"Manhattan","Gramercy","Yellow Zone" +108,"Brooklyn","Gravesend","Boro Zone" +109,"Staten Island","Great Kills","Boro Zone" +110,"Staten Island","Great Kills Park","Boro Zone" +111,"Brooklyn","Green-Wood Cemetery","Boro Zone" +112,"Brooklyn","Greenpoint","Boro Zone" +113,"Manhattan","Greenwich Village North","Yellow Zone" +114,"Manhattan","Greenwich Village South","Yellow Zone" +115,"Staten Island","Grymes Hill/Clifton","Boro Zone" +116,"Manhattan","Hamilton Heights","Boro Zone" +117,"Queens","Hammels/Arverne","Boro Zone" +118,"Staten Island","Heartland Village/Todt Hill","Boro Zone" +119,"Bronx","Highbridge","Boro Zone" +120,"Manhattan","Highbridge Park","Boro Zone" +121,"Queens","Hillcrest/Pomonok","Boro Zone" +122,"Queens","Hollis","Boro Zone" +123,"Brooklyn","Homecrest","Boro Zone" +124,"Queens","Howard Beach","Boro Zone" +125,"Manhattan","Hudson Sq","Yellow Zone" +126,"Bronx","Hunts Point","Boro Zone" +127,"Manhattan","Inwood","Boro Zone" +128,"Manhattan","Inwood Hill Park","Boro Zone" +129,"Queens","Jackson Heights","Boro Zone" +130,"Queens","Jamaica","Boro Zone" +131,"Queens","Jamaica Estates","Boro Zone" +132,"Queens","JFK Airport","Airports" +133,"Brooklyn","Kensington","Boro Zone" +134,"Queens","Kew Gardens","Boro Zone" +135,"Queens","Kew Gardens Hills","Boro Zone" +136,"Bronx","Kingsbridge Heights","Boro Zone" +137,"Manhattan","Kips Bay","Yellow Zone" +138,"Queens","LaGuardia Airport","Airports" +139,"Queens","Laurelton","Boro Zone" +140,"Manhattan","Lenox Hill East","Yellow Zone" +141,"Manhattan","Lenox Hill West","Yellow Zone" +142,"Manhattan","Lincoln Square East","Yellow Zone" +143,"Manhattan","Lincoln Square West","Yellow Zone" +144,"Manhattan","Little Italy/NoLiTa","Yellow Zone" +145,"Queens","Long Island City/Hunters Point","Boro Zone" +146,"Queens","Long Island City/Queens Plaza","Boro Zone" +147,"Bronx","Longwood","Boro Zone" +148,"Manhattan","Lower East Side","Yellow Zone" +149,"Brooklyn","Madison","Boro Zone" +150,"Brooklyn","Manhattan Beach","Boro Zone" +151,"Manhattan","Manhattan Valley","Yellow Zone" +152,"Manhattan","Manhattanville","Boro Zone" +153,"Manhattan","Marble Hill","Boro Zone" +154,"Brooklyn","Marine Park/Floyd Bennett Field","Boro Zone" +155,"Brooklyn","Marine Park/Mill Basin","Boro Zone" +156,"Staten Island","Mariners Harbor","Boro Zone" +157,"Queens","Maspeth","Boro Zone" +158,"Manhattan","Meatpacking/West Village West","Yellow Zone" +159,"Bronx","Melrose South","Boro Zone" +160,"Queens","Middle Village","Boro Zone" +161,"Manhattan","Midtown Center","Yellow Zone" +162,"Manhattan","Midtown East","Yellow Zone" +163,"Manhattan","Midtown North","Yellow Zone" +164,"Manhattan","Midtown South","Yellow Zone" +165,"Brooklyn","Midwood","Boro Zone" +166,"Manhattan","Morningside Heights","Boro Zone" +167,"Bronx","Morrisania/Melrose","Boro Zone" +168,"Bronx","Mott Haven/Port Morris","Boro Zone" +169,"Bronx","Mount Hope","Boro Zone" +170,"Manhattan","Murray Hill","Yellow Zone" +171,"Queens","Murray Hill-Queens","Boro Zone" +172,"Staten Island","New Dorp/Midland Beach","Boro Zone" +173,"Queens","North Corona","Boro Zone" +174,"Bronx","Norwood","Boro Zone" +175,"Queens","Oakland Gardens","Boro Zone" +176,"Staten Island","Oakwood","Boro Zone" +177,"Brooklyn","Ocean Hill","Boro Zone" +178,"Brooklyn","Ocean Parkway South","Boro Zone" +179,"Queens","Old Astoria","Boro Zone" +180,"Queens","Ozone Park","Boro Zone" +181,"Brooklyn","Park Slope","Boro Zone" +182,"Bronx","Parkchester","Boro Zone" +183,"Bronx","Pelham Bay","Boro Zone" +184,"Bronx","Pelham Bay Park","Boro Zone" +185,"Bronx","Pelham Parkway","Boro Zone" +186,"Manhattan","Penn Station/Madison Sq West","Yellow Zone" +187,"Staten Island","Port Richmond","Boro Zone" +188,"Brooklyn","Prospect-Lefferts Gardens","Boro Zone" +189,"Brooklyn","Prospect Heights","Boro Zone" +190,"Brooklyn","Prospect Park","Boro Zone" +191,"Queens","Queens Village","Boro Zone" +192,"Queens","Queensboro Hill","Boro Zone" +193,"Queens","Queensbridge/Ravenswood","Boro Zone" +194,"Manhattan","Randalls Island","Yellow Zone" +195,"Brooklyn","Red Hook","Boro Zone" +196,"Queens","Rego Park","Boro Zone" +197,"Queens","Richmond Hill","Boro Zone" +198,"Queens","Ridgewood","Boro Zone" +199,"Bronx","Rikers Island","Boro Zone" +200,"Bronx","Riverdale/North Riverdale/Fieldston","Boro Zone" +201,"Queens","Rockaway Park","Boro Zone" +202,"Manhattan","Roosevelt Island","Boro Zone" +203,"Queens","Rosedale","Boro Zone" +204,"Staten Island","Rossville/Woodrow","Boro Zone" +205,"Queens","Saint Albans","Boro Zone" +206,"Staten Island","Saint George/New Brighton","Boro Zone" +207,"Queens","Saint Michaels Cemetery/Woodside","Boro Zone" +208,"Bronx","Schuylerville/Edgewater Park","Boro Zone" +209,"Manhattan","Seaport","Yellow Zone" +210,"Brooklyn","Sheepshead Bay","Boro Zone" +211,"Manhattan","SoHo","Yellow Zone" +212,"Bronx","Soundview/Bruckner","Boro Zone" +213,"Bronx","Soundview/Castle Hill","Boro Zone" +214,"Staten Island","South Beach/Dongan Hills","Boro Zone" +215,"Queens","South Jamaica","Boro Zone" +216,"Queens","South Ozone Park","Boro Zone" +217,"Brooklyn","South Williamsburg","Boro Zone" +218,"Queens","Springfield Gardens North","Boro Zone" +219,"Queens","Springfield Gardens South","Boro Zone" +220,"Bronx","Spuyten Duyvil/Kingsbridge","Boro Zone" +221,"Staten Island","Stapleton","Boro Zone" +222,"Brooklyn","Starrett City","Boro Zone" +223,"Queens","Steinway","Boro Zone" +224,"Manhattan","Stuy Town/Peter Cooper Village","Yellow Zone" +225,"Brooklyn","Stuyvesant Heights","Boro Zone" +226,"Queens","Sunnyside","Boro Zone" +227,"Brooklyn","Sunset Park East","Boro Zone" +228,"Brooklyn","Sunset Park West","Boro Zone" +229,"Manhattan","Sutton Place/Turtle Bay North","Yellow Zone" +230,"Manhattan","Times Sq/Theatre District","Yellow Zone" +231,"Manhattan","TriBeCa/Civic Center","Yellow Zone" +232,"Manhattan","Two Bridges/Seward Park","Yellow Zone" +233,"Manhattan","UN/Turtle Bay South","Yellow Zone" +234,"Manhattan","Union Sq","Yellow Zone" +235,"Bronx","University Heights/Morris Heights","Boro Zone" +236,"Manhattan","Upper East Side North","Yellow Zone" +237,"Manhattan","Upper East Side South","Yellow Zone" +238,"Manhattan","Upper West Side North","Yellow Zone" +239,"Manhattan","Upper West Side South","Yellow Zone" +240,"Bronx","Van Cortlandt Park","Boro Zone" +241,"Bronx","Van Cortlandt Village","Boro Zone" +242,"Bronx","Van Nest/Morris Park","Boro Zone" +243,"Manhattan","Washington Heights North","Boro Zone" +244,"Manhattan","Washington Heights South","Boro Zone" +245,"Staten Island","West Brighton","Boro Zone" +246,"Manhattan","West Chelsea/Hudson Yards","Yellow Zone" +247,"Bronx","West Concourse","Boro Zone" +248,"Bronx","West Farms/Bronx River","Boro Zone" +249,"Manhattan","West Village","Yellow Zone" +250,"Bronx","Westchester Village/Unionport","Boro Zone" +251,"Staten Island","Westerleigh","Boro Zone" +252,"Queens","Whitestone","Boro Zone" +253,"Queens","Willets Point","Boro Zone" +254,"Bronx","Williamsbridge/Olinville","Boro Zone" +255,"Brooklyn","Williamsburg (North Side)","Boro Zone" +256,"Brooklyn","Williamsburg (South Side)","Boro Zone" +257,"Brooklyn","Windsor Terrace","Boro Zone" +258,"Queens","Woodhaven","Boro Zone" +259,"Bronx","Woodlawn/Wakefield","Boro Zone" +260,"Queens","Woodside","Boro Zone" +261,"Manhattan","World Trade Center","Yellow Zone" +262,"Manhattan","Yorkville East","Yellow Zone" +263,"Manhattan","Yorkville West","Yellow Zone" +264,"Unknown","NV","N/A" +265,"Unknown","NA","N/A" diff --git a/tests/clickhouse/testdata/migrations-remote/00001_a.sql b/tests/clickhouse/testdata/migrations-remote/00001_a.sql new file mode 100644 index 0000000..858370c --- /dev/null +++ b/tests/clickhouse/testdata/migrations-remote/00001_a.sql @@ -0,0 +1,17 @@ +-- +goose Up +CREATE DICTIONARY taxi_zone_dictionary ( + LocationID UInt16 DEFAULT 0, + Borough String, + Zone String, + service_zone String +) +PRIMARY KEY LocationID +SOURCE(HTTP( + url 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/taxi_zone_lookup.csv' + format 'CSVWithNames' +)) +LIFETIME(0) +LAYOUT(HASHED()); + +-- +goose Down +DROP DICTIONARY IF EXISTS taxi_zone_dictionary; diff --git a/tests/clickhouse/testdata/migrations/00001_a.sql b/tests/clickhouse/testdata/migrations/00001_a.sql new file mode 100644 index 0000000..8568770 --- /dev/null +++ b/tests/clickhouse/testdata/migrations/00001_a.sql @@ -0,0 +1,55 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS trips +( + `trip_id` UInt32, + `vendor_id` Enum8('1' = 1, '2' = 2, '3' = 3, '4' = 4, 'CMT' = 5, 'VTS' = 6, 'DDS' = 7, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14, '' = 15), + `pickup_date` Date, + `pickup_datetime` DateTime, + `dropoff_date` Date, + `dropoff_datetime` DateTime, + `store_and_fwd_flag` UInt8, + `rate_code_id` UInt8, + `pickup_longitude` Float64, + `pickup_latitude` Float64, + `dropoff_longitude` Float64, + `dropoff_latitude` Float64, + `passenger_count` UInt8, + `trip_distance` Float64, + `fare_amount` Float32, + `extra` Float32, + `mta_tax` Float32, + `tip_amount` Float32, + `tolls_amount` Float32, + `ehail_fee` Float32, + `improvement_surcharge` Float32, + `total_amount` Float32, + `payment_type` Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), + `trip_type` UInt8, + `pickup` FixedString(25), + `dropoff` FixedString(25), + `cab_type` Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), + `pickup_nyct2010_gid` Int8, + `pickup_ctlabel` Float32, + `pickup_borocode` Int8, + `pickup_ct2010` String, + `pickup_boroct2010` FixedString(7), + `pickup_cdeligibil` String, + `pickup_ntacode` FixedString(4), + `pickup_ntaname` String, + `pickup_puma` UInt16, + `dropoff_nyct2010_gid` UInt8, + `dropoff_ctlabel` Float32, + `dropoff_borocode` UInt8, + `dropoff_ct2010` String, + `dropoff_boroct2010` FixedString(7), + `dropoff_cdeligibil` String, + `dropoff_ntacode` FixedString(4), + `dropoff_ntaname` String, + `dropoff_puma` UInt16 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(pickup_date) +ORDER BY pickup_datetime; + +-- +goose Down +DROP TABLE IF EXISTS trips; diff --git a/tests/clickhouse/testdata/migrations/00002_b.sql b/tests/clickhouse/testdata/migrations/00002_b.sql new file mode 100644 index 0000000..2f34bf0 --- /dev/null +++ b/tests/clickhouse/testdata/migrations/00002_b.sql @@ -0,0 +1,13 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS clickstream ( + customer_id String, + time_stamp Date, + click_event_type String, + country_code FixedString(2), + source_id UInt64 +) +ENGINE = MergeTree() +ORDER BY (time_stamp); + +-- +goose Down +DROP TABLE IF EXISTS clickstream; diff --git a/tests/clickhouse/testdata/migrations/00003_c.sql b/tests/clickhouse/testdata/migrations/00003_c.sql new file mode 100644 index 0000000..472880e --- /dev/null +++ b/tests/clickhouse/testdata/migrations/00003_c.sql @@ -0,0 +1,8 @@ +-- +goose Up +INSERT INTO clickstream VALUES ('customer1', '2021-10-02', 'add_to_cart', 'US', 568239 ); + +INSERT INTO clickstream (customer_id, time_stamp, click_event_type) VALUES ('customer2', '2021-10-30', 'remove_from_cart' ); + +INSERT INTO clickstream (* EXCEPT(country_code)) VALUES ('customer3', '2021-11-07', 'checkout', 307493 ); + +-- +goose Down