USEFULL
/
gogs
mirror of https://github.com/gogs/gogs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

schemadoc: add go:generate to output database schema (#6310) 

* schemadoc: add go:generate to output database schema

* Check errors

* Revert string renames
pull/6324/head
ᴜɴᴋɴᴡᴏɴ 2020-09-06 17:02:25 +08:00 committed by GitHub
parent 519e59b577
commit 06193ed825
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 221 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Taskfile.yml
View File

@ -27,7 +27,7 @@ tasks:
BUILD_COMMIT:
sh: git rev-parse HEAD
generate:
generate-bindata:
desc: Generate bindata for all assets.
deps: [clean]
cmds:
@ -35,6 +35,15 @@ tasks:
- go generate internal/assets/templates/templates.go
- go generate internal/assets/public/public.go
generate-schemadoc:
desc: Generate database schema documentation.
cmds:
- go generate ./internal/db/schemadoc
generate:
desc: Run all go:generate commands.
deps: [generate-bindata, generate-schemadoc]
test:
desc: Run all tests.
cmds:

46
docs/dev/database_schema.md Normal file
View File

@ -0,0 +1,46 @@
# Table "access_token"
```
FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3
+-------------+--------------+--------------------+-----------------------+--------------------+
ID | id | BIGSERIAL | BIGINT AUTO_INCREMENT | INTEGER
UserID | uid | BIGINT | BIGINT | INTEGER
Name | name | TEXT | LONGTEXT | TEXT
Sha1 | sha1 | VARCHAR(40) UNIQUE | VARCHAR(40) UNIQUE | VARCHAR(40) UNIQUE
CreatedUnix | created_unix | BIGINT | BIGINT | INTEGER
UpdatedUnix | updated_unix | BIGINT | BIGINT | INTEGER
Primary keys: id
```
# Table "lfs_object"
```
FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3
+-----------+------------+----------------------+----------------------+-------------------+
RepoID | repo_id | BIGINT | BIGINT | INTEGER
OID | oid | TEXT | VARCHAR(191) | TEXT
Size | size | BIGINT NOT NULL | BIGINT NOT NULL | INTEGER NOT NULL
Storage | storage | TEXT NOT NULL | LONGTEXT NOT NULL | TEXT NOT NULL
CreatedAt | created_at | TIMESTAMPTZ NOT NULL | DATETIME(3) NOT NULL | DATETIME NOT NULL
Primary keys: repo_id, oid
```
# Table "login_source"
```
FIELD | COLUMN | POSTGRESQL | MYSQL | SQLITE3
+-------------+--------------+------------------+-----------------------+------------------+
ID | id | BIGSERIAL | BIGINT AUTO_INCREMENT | INTEGER
Type | type | BIGINT | BIGINT | INTEGER
Name | name | TEXT UNIQUE | VARCHAR(191) UNIQUE | TEXT UNIQUE
IsActived | is_actived | BOOLEAN NOT NULL | BOOLEAN NOT NULL | NUMERIC NOT NULL
IsDefault | is_default | BOOLEAN | BOOLEAN | NUMERIC
RawConfig | cfg | TEXT | TEXT | TEXT
CreatedUnix | created_unix | BIGINT | BIGINT | INTEGER
UpdatedUnix | updated_unix | BIGINT | BIGINT | INTEGER
Primary keys: id
```

2
docs/dev/import_locale.md
View File

@ -9,7 +9,7 @@
Locale files has been successfully imported!
```
1. Run `task generate` to generate corresponding bindata.
1. Run `task generate-bindata` to generate corresponding bindata.
1. Run `task web` to start the web server, then visit the site in the browser to make sure nothing blows up.
1. Check out a new branch using `git checkout -b update-locales`.
1. Stash changes then run `git commit -m "locale: sync from Crowdin"`.

3
go.mod
View File

@ -35,7 +35,7 @@ require (
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/niklasfasching/go-org v0.1.9
github.com/olekukonko/tablewriter v0.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.1
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.2.0
github.com/prometheus/client_golang v1.6.0
@ -53,6 +53,7 @@ require (
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
golang.org/x/text v0.3.3
gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df

12
go.sum
View File

@ -44,9 +44,7 @@ github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGii
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg=
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@ -55,8 +53,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
github.com/editorconfig/editorconfig-core-go/v2 v2.3.2 h1:j9GLz0kWF9+1T3IX0MOhhvzLtqhFOvIKLhZFxtY95Qc=
github.com/editorconfig/editorconfig-core-go/v2 v2.3.2/go.mod h1:+u4rFiKVvlbukHyJM76GYXqQcnHScxvQCuKpMLRtJVw=
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@ -203,11 +199,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43 h1:jTkyeF7NZ5oIr0ESmcrpiDgAfoidCBF4F5kJhjtaRwE=
github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE=
github.com/jinzhu/gorm v1.9.11/go.mod h1:bu/pK8szGZ2puuErfU0RwyeNdsf3e6nCX/noXaVxkfw=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@ -236,7 +229,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@ -261,7 +253,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@ -402,7 +393,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -509,6 +499,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0 h1:/21c4hNFgj8A1D54vgJZwQlywp64/RUBHzlPdpy5h4s=
gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0/go.mod h1:0uueny64T996pN6bez2N3S8HWyPcpyfTPma8Wc1Awx4=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=

4
internal/db/backup.go
View File

@ -40,7 +40,7 @@ func DumpDatabase(db *gorm.DB, dirPath string, verbose bool) error {
return errors.Wrap(err, "dump legacy tables")
}
for _, table := range tables {
for _, table := range Tables {
tableName := getTableType(table)
if verbose {
log.Trace("Dumping table %q...", tableName)
@ -125,7 +125,7 @@ func ImportDatabase(db *gorm.DB, dirPath string, verbose bool) error {
return errors.Wrap(err, "import legacy tables")
}
for _, table := range tables {
for _, table := range Tables {
tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
err := func() error {
tableFile := filepath.Join(dirPath, tableName+".json")

10
internal/db/backup_test.go
View File

@ -26,11 +26,11 @@ func Test_dumpAndImport(t *testing.T) {
t.Parallel()
if len(tables) != 3 {
t.Fatalf("New table has added (want 3 got %d), please add new tests for the table and update this check", len(tables))
if len(Tables) != 3 {
t.Fatalf("New table has added (want 3 got %d), please add new tests for the table and update this check", len(Tables))
}
db := initTestDB(t, "dumpAndImport", tables...)
db := initTestDB(t, "dumpAndImport", Tables...)
setupDBToDump(t, db)
dumpTables(t, db)
importTables(t, db)
@ -109,7 +109,7 @@ func setupDBToDump(t *testing.T, db *gorm.DB) {
func dumpTables(t *testing.T, db *gorm.DB) {
t.Helper()
for _, table := range tables {
for _, table := range Tables {
tableName := getTableType(table)
var buf bytes.Buffer
@ -126,7 +126,7 @@ func dumpTables(t *testing.T, db *gorm.DB) {
func importTables(t *testing.T, db *gorm.DB) {
t.Helper()
for _, table := range tables {
for _, table := range Tables {
tableName := getTableType(table)
err := func() error {

6
internal/db/db.go
View File

@ -137,8 +137,10 @@ func openDB(opts conf.DatabaseOpts, cfg *gorm.Config) (*gorm.DB, error) {
return gorm.Open(dialector, cfg)
}
// Tables is the list of struct-to-table mappings.
//
// NOTE: Lines are sorted in alphabetical order, each letter in its own line.
var tables = []interface{}{
var Tables = []interface{}{
new(AccessToken),
new(LFSObject), new(LoginSource),
}
@ -196,7 +198,7 @@ func Init() (*gorm.DB, error) {
// NOTE: GORM has problem detecting existing columns, see https://github.com/gogs/gogs/issues/6091.
// Therefore only use it to create new tables, and do customized migration with future changes.
for _, table := range tables {
for _, table := range Tables {
if db.Migrator().HasTable(table) {
continue
}

149
internal/db/schemadoc/main.go Normal file
View File

@ -0,0 +1,149 @@
package main
import (
"log"
"os"
"strings"
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"gopkg.in/DATA-DOG/go-sqlmock.v2"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"gorm.io/gorm/schema"
"gogs.io/gogs/internal/db"
)
//go:generate go run main.go ../../../docs/dev/database_schema.md
func main() {
w, err := os.Create(os.Args[1])
if err != nil {
log.Fatalf("Failed to create file: %v", err)
}
defer func() { _ = w.Close() }()
conn, _, err := sqlmock.New()
if err != nil {
log.Fatalf("Failed to get mock connection: %v", err)
}
defer func() { _ = conn.Close() }()
dialectors := []gorm.Dialector{
postgres.New(postgres.Config{
Conn: conn,
}),
mysql.New(mysql.Config{
Conn: conn,
SkipInitializeWithVersion: true,
}),
sqlite.Open(""),
}
collected := make([][]*tableInfo, 0, len(dialectors))
for i, dialector := range dialectors {
tableInfos, err := generate(dialector)
if err != nil {
log.Fatalf("Failed to get table info of %d: %v", i, err)
}
collected = append(collected, tableInfos)
}
for i, ti := range collected[0] {
_, _ = w.WriteString(`# Table "` + ti.Name + `"`)
_, _ = w.WriteString("\n\n")
_, _ = w.WriteString("```\n")
table := tablewriter.NewWriter(w)
table.SetHeader([]string{"Field", "Column", "PostgreSQL", "MySQL", "SQLite3"})
table.SetBorder(false)
for j, f := range ti.Fields {
table.Append([]string{
f.Name, f.Column,
strings.ToUpper(f.Type), // PostgreSQL
strings.ToUpper(collected[1][i].Fields[j].Type), // MySQL
strings.ToUpper(collected[2][i].Fields[j].Type), // SQLite3
})
}
table.Render()
_, _ = w.WriteString("\n")
_, _ = w.WriteString("Primary keys: ")
_, _ = w.WriteString(strings.Join(ti.PrimaryKeys, ", "))
_, _ = w.WriteString("\n")
_, _ = w.WriteString("```\n\n")
}
}
type tableField struct {
Name string
Column string
Type string
}
type tableInfo struct {
Name string
Fields []*tableField
PrimaryKeys []string
}
// This function is derived from gorm.io/gorm/migrator/migrator.go:Migrator.CreateTable.
func generate(dialector gorm.Dialector) ([]*tableInfo, error) {
conn, err := gorm.Open(dialector, &gorm.Config{
NamingStrategy: schema.NamingStrategy{
SingularTable: true,
},
DryRun: true,
DisableAutomaticPing: true,
})
if err != nil {
return nil, errors.Wrap(err, "open database")
}
m := conn.Migrator().(interface {
RunWithValue(value interface{}, fc func(*gorm.Statement) error) error
FullDataTypeOf(*schema.Field) clause.Expr
})
tableInfos := make([]*tableInfo, 0, len(db.Tables))
for _, table := range db.Tables {
err = m.RunWithValue(table, func(stmt *gorm.Statement) error {
fields := make([]*tableField, 0, len(stmt.Schema.DBNames))
for _, field := range stmt.Schema.Fields {
if field.DBName == "" {
continue
}
fields = append(fields, &tableField{
Name: field.Name,
Column: field.DBName,
Type: m.FullDataTypeOf(field).SQL,
})
}
primaryKeys := make([]string, 0, len(stmt.Schema.PrimaryFields))
if len(stmt.Schema.PrimaryFields) > 0 {
for _, field := range stmt.Schema.PrimaryFields {
primaryKeys = append(primaryKeys, field.DBName)
}
}
tableInfos = append(tableInfos, &tableInfo{
Name: stmt.Table,
Fields: fields,
PrimaryKeys: primaryKeys,
})
return nil
})
if err != nil {
return nil, errors.Wrap(err, "gather table information")
}
}
return tableInfos, nil
}