diff --git a/Taskfile.yml b/Taskfile.yml index f4fafa024..189a0d736 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,7 +27,7 @@ tasks: BUILD_COMMIT: sh: git rev-parse HEAD - generate: + generate-bindata: desc: Generate bindata for all assets. deps: [clean] cmds: @@ -35,6 +35,15 @@ tasks: - go generate internal/assets/templates/templates.go - go generate internal/assets/public/public.go + generate-schemadoc: + desc: Generate database schema documentation. + cmds: + - go generate ./internal/db/schemadoc + + generate: + desc: Run all go:generate commands. + deps: [generate-bindata, generate-schemadoc] + test: desc: Run all tests. cmds: diff --git a/docs/dev/database_schema.md b/docs/dev/database_schema.md new file mode 100644 index 000000000..39e1be77a --- /dev/null +++ b/docs/dev/database_schema.md @@ -0,0 +1,46 @@ +# Table "access_token" + +``` + FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3 ++-------------+--------------+--------------------+-----------------------+--------------------+ + ID | id | BIGSERIAL | BIGINT AUTO_INCREMENT | INTEGER + UserID | uid | BIGINT | BIGINT | INTEGER + Name | name | TEXT | LONGTEXT | TEXT + Sha1 | sha1 | VARCHAR(40) UNIQUE | VARCHAR(40) UNIQUE | VARCHAR(40) UNIQUE + CreatedUnix | created_unix | BIGINT | BIGINT | INTEGER + UpdatedUnix | updated_unix | BIGINT | BIGINT | INTEGER + +Primary keys: id +``` + +# Table "lfs_object" + +``` + FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3 ++-----------+------------+----------------------+----------------------+-------------------+ + RepoID | repo_id | BIGINT | BIGINT | INTEGER + OID | oid | TEXT | VARCHAR(191) | TEXT + Size | size | BIGINT NOT NULL | BIGINT NOT NULL | INTEGER NOT NULL + Storage | storage | TEXT NOT NULL | LONGTEXT NOT NULL | TEXT NOT NULL + CreatedAt | created_at | TIMESTAMPTZ NOT NULL | DATETIME(3) NOT NULL | DATETIME NOT NULL + +Primary keys: repo_id, oid +``` + +# Table "login_source" + +``` + FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3 ++-------------+--------------+------------------+-----------------------+------------------+ + ID | id | BIGSERIAL | BIGINT AUTO_INCREMENT | INTEGER + Type | type | BIGINT | BIGINT | INTEGER + Name | name | TEXT UNIQUE | VARCHAR(191) UNIQUE | TEXT UNIQUE + IsActived | is_actived | BOOLEAN NOT NULL | BOOLEAN NOT NULL | NUMERIC NOT NULL + IsDefault | is_default | BOOLEAN | BOOLEAN | NUMERIC + RawConfig | cfg | TEXT | TEXT | TEXT + CreatedUnix | created_unix | BIGINT | BIGINT | INTEGER + UpdatedUnix | updated_unix | BIGINT | BIGINT | INTEGER + +Primary keys: id +``` + diff --git a/docs/dev/import_locale.md b/docs/dev/import_locale.md index 156cedc54..ab93a92c5 100644 --- a/docs/dev/import_locale.md +++ b/docs/dev/import_locale.md @@ -9,7 +9,7 @@ Locale files has been successfully imported! ``` -1. Run `task generate` to generate corresponding bindata. +1. Run `task generate-bindata` to generate corresponding bindata. 1. Run `task web` to start the web server, then visit the site in the browser to make sure nothing blows up. 1. Check out a new branch using `git checkout -b update-locales`. 1. Stash changes then run `git commit -m "locale: sync from Crowdin"`. diff --git a/go.mod b/go.mod index 770773cb6..ab238606e 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/niklasfasching/go-org v0.1.9 - github.com/olekukonko/tablewriter v0.0.1 // indirect + github.com/olekukonko/tablewriter v0.0.1 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 github.com/prometheus/client_golang v1.6.0 @@ -53,6 +53,7 @@ require ( golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e golang.org/x/text v0.3.3 + gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df diff --git a/go.sum b/go.sum index facec2306..ef200a619 100644 --- a/go.sum +++ b/go.sum @@ -44,9 +44,7 @@ github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGii github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -55,8 +53,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/editorconfig/editorconfig-core-go/v2 v2.3.2 h1:j9GLz0kWF9+1T3IX0MOhhvzLtqhFOvIKLhZFxtY95Qc= github.com/editorconfig/editorconfig-core-go/v2 v2.3.2/go.mod h1:+u4rFiKVvlbukHyJM76GYXqQcnHScxvQCuKpMLRtJVw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -203,11 +199,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43 h1:jTkyeF7NZ5oIr0ESmcrpiDgAfoidCBF4F5kJhjtaRwE= github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= -github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE= -github.com/jinzhu/gorm v1.9.11/go.mod h1:bu/pK8szGZ2puuErfU0RwyeNdsf3e6nCX/noXaVxkfw= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -236,7 +229,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -261,7 +253,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= -github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -402,7 +393,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -509,6 +499,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0 h1:/21c4hNFgj8A1D54vgJZwQlywp64/RUBHzlPdpy5h4s= +gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0/go.mod h1:0uueny64T996pN6bez2N3S8HWyPcpyfTPma8Wc1Awx4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= diff --git a/internal/db/backup.go b/internal/db/backup.go index 985b3c0f8..2f76bb04f 100644 --- a/internal/db/backup.go +++ b/internal/db/backup.go @@ -40,7 +40,7 @@ func DumpDatabase(db *gorm.DB, dirPath string, verbose bool) error { return errors.Wrap(err, "dump legacy tables") } - for _, table := range tables { + for _, table := range Tables { tableName := getTableType(table) if verbose { log.Trace("Dumping table %q...", tableName) @@ -125,7 +125,7 @@ func ImportDatabase(db *gorm.DB, dirPath string, verbose bool) error { return errors.Wrap(err, "import legacy tables") } - for _, table := range tables { + for _, table := range Tables { tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.") err := func() error { tableFile := filepath.Join(dirPath, tableName+".json") diff --git a/internal/db/backup_test.go b/internal/db/backup_test.go index fafaf28ac..a65e6944f 100644 --- a/internal/db/backup_test.go +++ b/internal/db/backup_test.go @@ -26,11 +26,11 @@ func Test_dumpAndImport(t *testing.T) { t.Parallel() - if len(tables) != 3 { - t.Fatalf("New table has added (want 3 got %d), please add new tests for the table and update this check", len(tables)) + if len(Tables) != 3 { + t.Fatalf("New table has added (want 3 got %d), please add new tests for the table and update this check", len(Tables)) } - db := initTestDB(t, "dumpAndImport", tables...) + db := initTestDB(t, "dumpAndImport", Tables...) setupDBToDump(t, db) dumpTables(t, db) importTables(t, db) @@ -109,7 +109,7 @@ func setupDBToDump(t *testing.T, db *gorm.DB) { func dumpTables(t *testing.T, db *gorm.DB) { t.Helper() - for _, table := range tables { + for _, table := range Tables { tableName := getTableType(table) var buf bytes.Buffer @@ -126,7 +126,7 @@ func dumpTables(t *testing.T, db *gorm.DB) { func importTables(t *testing.T, db *gorm.DB) { t.Helper() - for _, table := range tables { + for _, table := range Tables { tableName := getTableType(table) err := func() error { diff --git a/internal/db/db.go b/internal/db/db.go index 03da40ca4..ce130f2cc 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -137,8 +137,10 @@ func openDB(opts conf.DatabaseOpts, cfg *gorm.Config) (*gorm.DB, error) { return gorm.Open(dialector, cfg) } +// Tables is the list of struct-to-table mappings. +// // NOTE: Lines are sorted in alphabetical order, each letter in its own line. -var tables = []interface{}{ +var Tables = []interface{}{ new(AccessToken), new(LFSObject), new(LoginSource), } @@ -196,7 +198,7 @@ func Init() (*gorm.DB, error) { // NOTE: GORM has problem detecting existing columns, see https://github.com/gogs/gogs/issues/6091. // Therefore only use it to create new tables, and do customized migration with future changes. - for _, table := range tables { + for _, table := range Tables { if db.Migrator().HasTable(table) { continue } diff --git a/internal/db/schemadoc/main.go b/internal/db/schemadoc/main.go new file mode 100644 index 000000000..3048f72fb --- /dev/null +++ b/internal/db/schemadoc/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "log" + "os" + "strings" + + "github.com/olekukonko/tablewriter" + "github.com/pkg/errors" + "gopkg.in/DATA-DOG/go-sqlmock.v2" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/schema" + + "gogs.io/gogs/internal/db" +) + +//go:generate go run main.go ../../../docs/dev/database_schema.md + +func main() { + w, err := os.Create(os.Args[1]) + if err != nil { + log.Fatalf("Failed to create file: %v", err) + } + defer func() { _ = w.Close() }() + + conn, _, err := sqlmock.New() + if err != nil { + log.Fatalf("Failed to get mock connection: %v", err) + } + defer func() { _ = conn.Close() }() + + dialectors := []gorm.Dialector{ + postgres.New(postgres.Config{ + Conn: conn, + }), + mysql.New(mysql.Config{ + Conn: conn, + SkipInitializeWithVersion: true, + }), + sqlite.Open(""), + } + collected := make([][]*tableInfo, 0, len(dialectors)) + for i, dialector := range dialectors { + tableInfos, err := generate(dialector) + if err != nil { + log.Fatalf("Failed to get table info of %d: %v", i, err) + } + + collected = append(collected, tableInfos) + } + + for i, ti := range collected[0] { + _, _ = w.WriteString(`# Table "` + ti.Name + `"`) + _, _ = w.WriteString("\n\n") + + _, _ = w.WriteString("```\n") + + table := tablewriter.NewWriter(w) + table.SetHeader([]string{"Field", "Column", "PostgreSQL", "MySQL", "SQLite3"}) + table.SetBorder(false) + for j, f := range ti.Fields { + table.Append([]string{ + f.Name, f.Column, + strings.ToUpper(f.Type), // PostgreSQL + strings.ToUpper(collected[1][i].Fields[j].Type), // MySQL + strings.ToUpper(collected[2][i].Fields[j].Type), // SQLite3 + }) + } + table.Render() + _, _ = w.WriteString("\n") + + _, _ = w.WriteString("Primary keys: ") + _, _ = w.WriteString(strings.Join(ti.PrimaryKeys, ", ")) + _, _ = w.WriteString("\n") + + _, _ = w.WriteString("```\n\n") + } +} + +type tableField struct { + Name string + Column string + Type string +} + +type tableInfo struct { + Name string + Fields []*tableField + PrimaryKeys []string +} + +// This function is derived from gorm.io/gorm/migrator/migrator.go:Migrator.CreateTable. +func generate(dialector gorm.Dialector) ([]*tableInfo, error) { + conn, err := gorm.Open(dialector, &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + SingularTable: true, + }, + DryRun: true, + DisableAutomaticPing: true, + }) + if err != nil { + return nil, errors.Wrap(err, "open database") + } + + m := conn.Migrator().(interface { + RunWithValue(value interface{}, fc func(*gorm.Statement) error) error + FullDataTypeOf(*schema.Field) clause.Expr + }) + tableInfos := make([]*tableInfo, 0, len(db.Tables)) + for _, table := range db.Tables { + err = m.RunWithValue(table, func(stmt *gorm.Statement) error { + fields := make([]*tableField, 0, len(stmt.Schema.DBNames)) + for _, field := range stmt.Schema.Fields { + if field.DBName == "" { + continue + } + + fields = append(fields, &tableField{ + Name: field.Name, + Column: field.DBName, + Type: m.FullDataTypeOf(field).SQL, + }) + } + + primaryKeys := make([]string, 0, len(stmt.Schema.PrimaryFields)) + if len(stmt.Schema.PrimaryFields) > 0 { + for _, field := range stmt.Schema.PrimaryFields { + primaryKeys = append(primaryKeys, field.DBName) + } + } + + tableInfos = append(tableInfos, &tableInfo{ + Name: stmt.Table, + Fields: fields, + PrimaryKeys: primaryKeys, + }) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "gather table information") + } + } + + return tableInfos, nil +}