test: add more parser tests (#445)

.gitignore

@@ -9,3 +9,4 @@
 
 # Local testing
 .envrc
+*.FAIL

Makefile

@@ -36,3 +36,7 @@ start-postgres:
 		-p ${GOOSE_POSTGRES_PORT}:5432 \
 		-l goose_test \
 		postgres:14-alpine
+
+.PHONY: clean
+clean:
+	@find . -type f -name '*.FAIL' -delete

internal/sqlparser/parser_test.go

@@ -2,9 +2,14 @@ package sqlparser
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/pressly/goose/v3/internal/check"
 )
 
 func TestSemicolons(t *testing.T) {
@@ -374,3 +379,104 @@ FOR EACH ROW EXECUTE PROCEDURE  update_updated_at_column();
 DROP TRIGGER update_properties_updated_at;
 DROP FUNCTION update_updated_at_column();
 `
+
+func TestValidUp(t *testing.T) {
+	t.Parallel()
+	// Test valid "up" parser logic.
+	//
+	// This test expects each directory, such as: internal/sqlparser/testdata/valid-up/test01
+	//
+	// to contain exactly one migration file called "input.sql". We read this file and pass it
+	// to the parser. Then we compare the statements against the golden files.
+	// Each golden file is equivalent to one statement.
+	//
+	// ├── 01.golden.sql
+	// ├── 02.golden.sql
+	// ├── 03.golden.sql
+	// └── input.sql
+	tests := []struct {
+		Name            string
+		StatementsCount int
+	}{
+		{Name: "test01", StatementsCount: 3},
+		{Name: "test02", StatementsCount: 1},
+		{Name: "test03", StatementsCount: 1},
+		{Name: "test04", StatementsCount: 2},
+		{Name: "test05", StatementsCount: 2},
+		{Name: "test06", StatementsCount: 3},
+	}
+	for _, tc := range tests {
+		path := filepath.Join("testdata", "valid-up", tc.Name)
+		t.Run(tc.Name, func(t *testing.T) {
+			testValidUp(t, path, tc.StatementsCount)
+		})
+	}
+}
+
+func testValidUp(t *testing.T, dir string, count int) {
+	t.Helper()
+
+	f, err := os.Open(filepath.Join(dir, "input.sql"))
+	check.NoError(t, err)
+	t.Cleanup(func() { f.Close() })
+	statements, _, err := ParseSQLMigration(f, true)
+	check.NoError(t, err)
+	check.Number(t, len(statements), count)
+	compareStatements(t, dir, statements)
+}
+
+func compareStatements(t *testing.T, dir string, statements []string) {
+	t.Helper()
+
+	files, err := os.ReadDir(dir)
+	check.NoError(t, err)
+	for _, goldenFile := range files {
+		if goldenFile.Name() == "input.sql" {
+			continue
+		}
+		if !strings.HasSuffix(goldenFile.Name(), ".golden.sql") {
+			t.Fatalf("expecting golden file with format <name>.golden.sql: got: %q. Try running `make clean` to remove previous failed files?", goldenFile.Name())
+		}
+		before, _, ok := cut(goldenFile.Name(), ".")
+		if !ok {
+			t.Fatal(`failed to cut on file delimiter ".", must be of the format NN.golden.sql`)
+		}
+		index, err := strconv.Atoi(before)
+		check.NoError(t, err)
+		index--
+
+		goldenFilePath := filepath.Join(dir, goldenFile.Name())
+		by, err := os.ReadFile(goldenFilePath)
+		check.NoError(t, err)
+
+		got, want := strings.TrimSpace(statements[index]), strings.TrimSpace(string(by))
+
+		if got != want {
+			if isCIEnvironment() {
+				t.Errorf("input does not match expected golden file:\n\ngot:\n%s\n\nwant:\n%s\n", got, want)
+			} else {
+				t.Error("input does not match expected output; diff files with .FAIL to debug")
+				t.Logf("\ndiff %v %v",
+					filepath.Join("internal", "sqlparser", goldenFilePath+".FAIL"),
+					filepath.Join("internal", "sqlparser", goldenFilePath),
+				)
+				err := ioutil.WriteFile(goldenFilePath+".FAIL", []byte(got+"\n"), 0644)
+				check.NoError(t, err)
+			}
+		}
+	}
+}
+
+// copied directly from strings.Cut (go1.18) to support older Go versions.
+// In the future, replace this with the upstream function.
+func cut(s, sep string) (before, after string, found bool) {
+	if i := strings.Index(s, sep); i >= 0 {
+		return s[:i], s[i+len(sep):], true
+	}
+	return s, "", false
+}
+
+func isCIEnvironment() bool {
+	ok, _ := strconv.ParseBool(os.Getenv("CI"))
+	return ok
+}

internal/sqlparser/testdata/valid-up/test01/01.golden.sql

@@ -0,0 +1,6 @@
+CREATE TABLE emp (
+    empname text,
+    salary integer,
+    last_date timestamp,
+    last_user text
+);

internal/sqlparser/testdata/valid-up/test01/02.golden.sql

@@ -0,0 +1,22 @@
+CREATE FUNCTION emp_stamp() RETURNS trigger AS $emp_stamp$
+    BEGIN
+        -- Check that empname and salary are given
+        IF NEW.empname IS NULL THEN
+            RAISE EXCEPTION 'empname cannot be null';
+        END IF;
+        IF NEW.salary IS NULL THEN
+            RAISE EXCEPTION '% cannot have null salary', NEW.empname;
+        END IF;
+
+        -- Who works for us when they must pay for it?
+        IF NEW.salary < 0 THEN
+            RAISE EXCEPTION '% cannot have a negative salary', NEW.empname;
+        END IF;
+
+        -- Remember who changed the payroll when
+        NEW.last_date := current_timestamp;
+        NEW.last_user := current_user;
+        RETURN NEW;
+    END;
+$emp_stamp$ LANGUAGE plpgsql;
+-- +goose StatementEnd

internal/sqlparser/testdata/valid-up/test01/03.golden.sql

@@ -0,0 +1,2 @@
+CREATE TRIGGER emp_stamp BEFORE INSERT OR UPDATE ON emp
+    FOR EACH ROW EXECUTE FUNCTION emp_stamp();

internal/sqlparser/testdata/valid-up/test01/input.sql

@@ -0,0 +1,36 @@
+-- +goose Up
+CREATE TABLE emp (
+    empname text,
+    salary integer,
+    last_date timestamp,
+    last_user text
+);
+
+-- +goose StatementBegin
+CREATE FUNCTION emp_stamp() RETURNS trigger AS $emp_stamp$
+    BEGIN
+        -- Check that empname and salary are given
+        IF NEW.empname IS NULL THEN
+            RAISE EXCEPTION 'empname cannot be null';
+        END IF;
+        IF NEW.salary IS NULL THEN
+            RAISE EXCEPTION '% cannot have null salary', NEW.empname;
+        END IF;
+
+        -- Who works for us when they must pay for it?
+        IF NEW.salary < 0 THEN
+            RAISE EXCEPTION '% cannot have a negative salary', NEW.empname;
+        END IF;
+
+        -- Remember who changed the payroll when
+        NEW.last_date := current_timestamp;
+        NEW.last_user := current_user;
+        RETURN NEW;
+    END;
+$emp_stamp$ LANGUAGE plpgsql;
+-- +goose StatementEnd
+
+CREATE TRIGGER emp_stamp BEFORE INSERT OR UPDATE ON emp
+    FOR EACH ROW EXECUTE FUNCTION emp_stamp();
+
+-- +goose Down

internal/sqlparser/testdata/valid-up/test02/01.golden.sql

@@ -0,0 +1,34 @@
+CREATE TABLE emp (
+    empname           text NOT NULL,
+    salary            integer
+);
+
+CREATE TABLE emp_audit(
+    operation         char(1)   NOT NULL,
+    stamp             timestamp NOT NULL,
+    userid            text      NOT NULL,
+    empname           text      NOT NULL,
+    salary integer
+);
+
+CREATE OR REPLACE FUNCTION process_emp_audit() RETURNS TRIGGER AS $emp_audit$
+    BEGIN
+        --
+        -- Create a row in emp_audit to reflect the operation performed on emp,
+        -- making use of the special variable TG_OP to work out the operation.
+        --
+        IF (TG_OP = 'DELETE') THEN
+            INSERT INTO emp_audit SELECT 'D', now(), user, OLD.*;
+        ELSIF (TG_OP = 'UPDATE') THEN
+            INSERT INTO emp_audit SELECT 'U', now(), user, NEW.*;
+        ELSIF (TG_OP = 'INSERT') THEN
+            INSERT INTO emp_audit SELECT 'I', now(), user, NEW.*;
+        END IF;
+        RETURN NULL; -- result is ignored since this is an AFTER trigger
+    END;
+$emp_audit$ LANGUAGE plpgsql;
+
+CREATE TRIGGER emp_audit
+AFTER INSERT OR UPDATE OR DELETE ON emp
+    FOR EACH ROW EXECUTE FUNCTION process_emp_audit();
+-- +goose StatementEnd

internal/sqlparser/testdata/valid-up/test02/input.sql

@@ -0,0 +1,38 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE emp (
+    empname           text NOT NULL,
+    salary            integer
+);
+
+CREATE TABLE emp_audit(
+    operation         char(1)   NOT NULL,
+    stamp             timestamp NOT NULL,
+    userid            text      NOT NULL,
+    empname           text      NOT NULL,
+    salary integer
+);
+
+CREATE OR REPLACE FUNCTION process_emp_audit() RETURNS TRIGGER AS $emp_audit$
+    BEGIN
+        --
+        -- Create a row in emp_audit to reflect the operation performed on emp,
+        -- making use of the special variable TG_OP to work out the operation.
+        --
+        IF (TG_OP = 'DELETE') THEN
+            INSERT INTO emp_audit SELECT 'D', now(), user, OLD.*;
+        ELSIF (TG_OP = 'UPDATE') THEN
+            INSERT INTO emp_audit SELECT 'U', now(), user, NEW.*;
+        ELSIF (TG_OP = 'INSERT') THEN
+            INSERT INTO emp_audit SELECT 'I', now(), user, NEW.*;
+        END IF;
+        RETURN NULL; -- result is ignored since this is an AFTER trigger
+    END;
+$emp_audit$ LANGUAGE plpgsql;
+
+CREATE TRIGGER emp_audit
+AFTER INSERT OR UPDATE OR DELETE ON emp
+    FOR EACH ROW EXECUTE FUNCTION process_emp_audit();
+-- +goose StatementEnd
+
+-- +goose Down

internal/sqlparser/testdata/valid-up/test03/01.golden.sql

@@ -0,0 +1,29 @@
+CREATE FUNCTION cs_update_referrer_type_proc() RETURNS INTEGER AS '
+DECLARE
+    referrer_keys RECORD;  -- Declare a generic record to be used in a FOR
+    a_output varchar(4000);
+BEGIN 
+    a_output := ''CREATE FUNCTION cs_find_referrer_type(varchar,varchar,varchar) 
+                  RETURNS VARCHAR AS '''' 
+                     DECLARE 
+                         v_host ALIAS FOR $1; 
+                         v_domain ALIAS FOR $2; 
+                         v_url ALIAS FOR $3;
+                     BEGIN ''; 
+
+
+    FOR referrer_keys IN SELECT * FROM cs_referrer_keys ORDER BY try_order LOOP
+        a_output := a_output || '' IF v_'' || referrer_keys.kind || '' LIKE '''''''''' 
+                 || referrer_keys.key_string || '''''''''' THEN RETURN '''''' 
+                 || referrer_keys.referrer_type || ''''''; END IF;''; 
+    END LOOP; 
+  
+    a_output := a_output || '' RETURN NULL; END; '''' LANGUAGE ''''plpgsql'''';''; 
+ 
+    -- This works because we are not substituting any variables
+    -- Otherwise it would fail. Look at PERFORM for another way to run functions
+    
+    EXECUTE a_output; 
+END; 
+' LANGUAGE 'plpgsql';
+-- +goose StatementEnd

internal/sqlparser/testdata/valid-up/test03/input.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE FUNCTION cs_update_referrer_type_proc() RETURNS INTEGER AS '
+DECLARE
+    referrer_keys RECORD;  -- Declare a generic record to be used in a FOR
+    a_output varchar(4000);
+BEGIN 
+    a_output := ''CREATE FUNCTION cs_find_referrer_type(varchar,varchar,varchar) 
+                  RETURNS VARCHAR AS '''' 
+                     DECLARE 
+                         v_host ALIAS FOR $1; 
+                         v_domain ALIAS FOR $2; 
+                         v_url ALIAS FOR $3;
+                     BEGIN ''; 
+
+-- 
+-- Notice how we scan through the results of a query in a FOR loop
+-- using the FOR <record> construct.
+--
+
+    FOR referrer_keys IN SELECT * FROM cs_referrer_keys ORDER BY try_order LOOP
+        a_output := a_output || '' IF v_'' || referrer_keys.kind || '' LIKE '''''''''' 
+                 || referrer_keys.key_string || '''''''''' THEN RETURN '''''' 
+                 || referrer_keys.referrer_type || ''''''; END IF;''; 
+    END LOOP; 
+  
+    a_output := a_output || '' RETURN NULL; END; '''' LANGUAGE ''''plpgsql'''';''; 
+ 
+    -- This works because we are not substituting any variables
+    -- Otherwise it would fail. Look at PERFORM for another way to run functions
+    
+    EXECUTE a_output; 
+END; 
+' LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose Down

internal/sqlparser/testdata/valid-up/test04/01.golden.sql

@@ -0,0 +1,4 @@
+CREATE TABLE ssh_keys (
+    id integer NOT NULL,
+    "publicKey" text
+);

internal/sqlparser/testdata/valid-up/test04/02.golden.sql

@@ -0,0 +1,8 @@
+INSERT INTO ssh_keys (id, "publicKey") VALUES (1, '-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAqAo9QORIXMPMa/qv8908Z2sH2+Xa/wITYTJQ2ojTZlgsQiQf85ifw3dgvVZK
+M7Zifl2NyVVCPb0hELr2JJla1u/1CgiuqDpcjP2cCu2YxB/JGyCvcon+3tETUz3Ri9NGzHCZ
+fkuWRZjkUvy7nfPLjzM+t6SEvY4lbn3ihLPumZjwgvuCY3vDZY8V1/NMoP8MKATGR+S7D7gv
+I6KD9jkiSsTJMiotb/dRkXE3bG0nmjchhhLzMG551G8IZEpWBHDqEisCIl8yCd9YZV69BZTu
+L48zPl/CFvA+KJJ6LklxfwWeVDQ+ve2OIW0B1uLhR/MsoYbDQztbgIayg6ieMO/KlQIDAQAB
+');
+-- +goose StatementEnd

internal/sqlparser/testdata/valid-up/test04/input.sql

@@ -0,0 +1,19 @@
+-- +goose Up
+
+CREATE TABLE ssh_keys (
+    id integer NOT NULL,
+    "publicKey" text
+);
+
+-- +goose StatementBegin
+INSERT INTO ssh_keys (id, "publicKey") VALUES (1, '-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAqAo9QORIXMPMa/qv8908Z2sH2+Xa/wITYTJQ2ojTZlgsQiQf85ifw3dgvVZK
+M7Zifl2NyVVCPb0hELr2JJla1u/1CgiuqDpcjP2cCu2YxB/JGyCvcon+3tETUz3Ri9NGzHCZ
+fkuWRZjkUvy7nfPLjzM+t6SEvY4lbn3ihLPumZjwgvuCY3vDZY8V1/NMoP8MKATGR+S7D7gv
+I6KD9jkiSsTJMiotb/dRkXE3bG0nmjchhhLzMG551G8IZEpWBHDqEisCIl8yCd9YZV69BZTu
+L48zPl/CFvA+KJJ6LklxfwWeVDQ+ve2OIW0B1uLhR/MsoYbDQztbgIayg6ieMO/KlQIDAQAB
+-----END RSA PUBLIC KEY-----
+');
+-- +goose StatementEnd
+
+-- +goose Down

internal/sqlparser/testdata/valid-up/test05/01.golden.sql

@@ -0,0 +1,4 @@
+CREATE TABLE ssh_keys (
+    id integer NOT NULL,
+    "publicKey" text
+);

internal/sqlparser/testdata/valid-up/test05/02.golden.sql

@@ -0,0 +1,6 @@
+CREATE TABLE ssh_keys_backup (
+    id integer NOT NULL,
+    -- insert comment here
+    "publicKey" text
+    -- insert comment there
+);

internal/sqlparser/testdata/valid-up/test05/input.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+
+CREATE TABLE ssh_keys (
+    id integer NOT NULL,
+    "publicKey" text
+-- insert comment there
+);
+-- insert comment there
+
+-- This is a dangling comment
+-- Another comment
+-- Foo comment
+
+CREATE TABLE ssh_keys_backup (
+    id integer NOT NULL,
+    -- insert comment here
+    "publicKey" text
+    -- insert comment there
+);
+
+
+-- +goose Down

internal/sqlparser/testdata/valid-up/test06/01.golden.sql

@@ -0,0 +1,3 @@
+CREATE TABLE article (
+    id text,
+            content text);

internal/sqlparser/testdata/valid-up/test06/02.golden.sql

@@ -0,0 +1,5 @@
+INSERT INTO article (id, content) VALUES ('id_0001',  E'# My markdown doc
+
+first paragraph
+
+second paragraph');

internal/sqlparser/testdata/valid-up/test06/03.golden.sql

@@ -0,0 +1,9 @@
+INSERT INTO article (id, content) VALUES ('id_0002',  E'# My second 
+
+markdown doc
+
+first paragraph
+
+    -- with an indent comment
+
+second paragraph');

internal/sqlparser/testdata/valid-up/test06/input.sql

@@ -0,0 +1,25 @@
+-- +goose Up
+
+CREATE TABLE article (
+    id text,
+            content text);
+
+INSERT INTO article (id, content) VALUES ('id_0001',  E'# My markdown doc
+
+first paragraph
+
+second paragraph');
+
+INSERT INTO article (id, content) VALUES ('id_0002',  E'# My second 
+
+markdown doc
+
+first paragraph
+
+-- with a comment
+    -- with an indent comment
+
+second paragraph');
+
+
+-- +goose Down