GOMODULES
/
godotenv
mirror of https://github.com/joho/godotenv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Multiline string support (#156) 

* refactor dotenv parser in order to support multi-line variable values declaration

Signed-off-by: x1unix <denis0051@gmail.com>

* Add multi-line var values test case and update comment test

Signed-off-by: x1unix <denis0051@gmail.com>

* Expand fixture tests to include multiline strings

* Update go versions to test against

* Switch to GOINSECURE for power8 CI task

* When tests fail, show source version of string (inc special chars)

* Update parser.go

Co-authored-by: Austin Sasko <austintyler0239@yahoo.com>

* Fix up bad merge

* Add a full fixture for comments for extra piece of mind

* Fix up some lint/staticcheck recommendations

* Test against go 1.19 too

Signed-off-by: x1unix <denis0051@gmail.com>
Co-authored-by: x1unix <denis0051@gmail.com>
Co-authored-by: Austin Sasko <austintyler0239@yahoo.com>
pull/140/merge 1.5.0-beta.0
John Barton 2023-01-27 13:14:16 +11:00 committed by GitHub
parent 0f21d20acb
commit cc9e9b7de7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 325 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -8,7 +8,7 @@ jobs:
strategy:
fail-fast: false
matrix:
go: [ '1.18', '1.17', '1.16', '1.15' ]
go: [ '1.19', '1.18', '1.17', '1.16', '1.15' ]
os: [ ubuntu-latest, macOS-latest, windows-latest ]
name: ${{ matrix.os }} Go ${{ matrix.go }} Tests
steps:

4
fixtures/comments.env Normal file
View File

@ -0,0 +1,4 @@
# Full line comment
foo=bar # baz
bar=foo#baz
baz="foo"#bar

10
fixtures/quoted.env
View File

@ -7,3 +7,13 @@ OPTION_F="2"
OPTION_G=""
OPTION_H="\n"
OPTION_I = "echo 'asd'"
OPTION_J='line 1
line 2'
OPTION_K='line one
this is \'quoted\'
one more line'
OPTION_L="line 1
line 2"
OPTION_M="line one
this is \"quoted\"
one more line"

72
godotenv.go
View File

@ -14,10 +14,10 @@
package godotenv
import (
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
@ -28,6 +28,16 @@ import (
const doubleQuoteSpecialChars = "\\\n\r\"!$`"
// Parse reads an env file from io.Reader, returning a map of keys and values.
func Parse(r io.Reader) (map[string]string, error) {
data, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return UnmarshalBytes(data)
}
// Load will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main).
@ -96,37 +106,17 @@ func Read(filenames ...string) (envMap map[string]string, err error) {
return
}
// Parse reads an env file from io.Reader, returning a map of keys and values.
func Parse(r io.Reader) (envMap map[string]string, err error) {
envMap = make(map[string]string)
var lines []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err = scanner.Err(); err != nil {
return
}
for _, fullLine := range lines {
if !isIgnoredLine(fullLine) {
var key, value string
key, value, err = parseLine(fullLine, envMap)
if err != nil {
return
}
envMap[key] = value
}
}
return
}
// Unmarshal reads an env file from a string, returning a map of keys and values.
func Unmarshal(str string) (envMap map[string]string, err error) {
return Parse(strings.NewReader(str))
return UnmarshalBytes([]byte(str))
}
// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values.
func UnmarshalBytes(src []byte) (map[string]string, error) {
out := make(map[string]string)
err := parseBytes(src, out)
return out, err
}
// Exec loads env vars from the specified filenames (empty map falls back to default)
@ -137,7 +127,9 @@ func Unmarshal(str string) (envMap map[string]string, err error) {
// If you want more fine grained control over your command it's recommended
// that you use `Load()` or `Read()` and the `os/exec` package yourself.
func Exec(filenames []string, cmd string, cmdArgs []string) error {
Load(filenames...)
if err := Load(filenames...); err != nil {
return err
}
command := exec.Command(cmd, cmdArgs...)
command.Stdin = os.Stdin
@ -161,8 +153,7 @@ func Write(envMap map[string]string, filename string) error {
if err != nil {
return err
}
file.Sync()
return err
return file.Sync()
}
// Marshal outputs the given environment as a dotenv-formatted environment file.
@ -202,7 +193,7 @@ func loadFile(filename string, overload bool) error {
for key, value := range envMap {
if !currentEnv[key] || overload {
os.Setenv(key, value)
_ = os.Setenv(key, value)
}
}
@ -259,15 +250,15 @@ func parseLine(line string, envMap map[string]string) (key string, value string,
}
if len(splitString) != 2 {
err = errors.New("Can't separate key from value")
err = errors.New("can't separate key from value")
return
}
// Parse the key
key = splitString[0]
if strings.HasPrefix(key, "export") {
key = strings.TrimPrefix(key, "export")
}
key = strings.TrimPrefix(key, "export")
key = strings.TrimSpace(key)
key = exportRegex.ReplaceAllString(splitString[0], "$1")
@ -343,11 +334,6 @@ func expandVariables(v string, m map[string]string) string {
})
}
func isIgnoredLine(line string) bool {
trimmedLine := strings.TrimSpace(line)
return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
}
func doubleQuoteEscape(line string) string {
for _, c := range doubleQuoteSpecialChars {
toReplace := "\\" + string(c)

100
godotenv_test.go
View File

@ -35,7 +35,7 @@ func loadEnvAndCompareValues(t *testing.T, loader func(files ...string) error, e
envValue := os.Getenv(k)
v := expectedValues[k]
if envValue != v {
t.Errorf("Mismatch for key '%v': expected '%v' got '%v'", k, v, envValue)
t.Errorf("Mismatch for key '%v': expected '%#v' got '%#v'", k, v, envValue)
}
}
}
@ -189,6 +189,10 @@ func TestLoadQuotedEnv(t *testing.T) {
"OPTION_G": "",
"OPTION_H": "\n",
"OPTION_I": "echo 'asd'",
"OPTION_J": "line 1\nline 2",
"OPTION_K": "line one\nthis is \\'quoted\\'\none more line",
"OPTION_L": "line 1\nline 2",
"OPTION_M": "line one\nthis is \"quoted\"\none more line",
}
loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets)
@ -271,6 +275,34 @@ func TestExpanding(t *testing.T) {
}
func TestVariableStringValueSeparator(t *testing.T) {
input := "TEST_URLS=\"stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443\""
want := map[string]string{
"TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443",
}
got, err := Parse(strings.NewReader(input))
if err != nil {
t.Error(err)
}
if len(got) != len(want) {
t.Fatalf(
"unexpected value:\nwant:\n\t%#v\n\ngot:\n\t%#v", want, got)
}
for k, wantVal := range want {
gotVal, ok := got[k]
if !ok {
t.Fatalf("key %q doesn't present in result", k)
}
if wantVal != gotVal {
t.Fatalf(
"mismatch in %q value:\nwant:\n\t%s\n\ngot:\n\t%s", k,
wantVal, gotVal)
}
}
}
func TestActualEnvVarsAreLeftAlone(t *testing.T) {
os.Clearenv()
os.Setenv("OPTION_A", "actualenv")
@ -377,33 +409,38 @@ func TestParsing(t *testing.T) {
}
func TestLinesToIgnore(t *testing.T) {
// it 'ignores empty lines' do
// expect(env("\n \t \nfoo=bar\n \nfizz=buzz")).to eql('foo' => 'bar', 'fizz' => 'buzz')
if !isIgnoredLine("\n") {
t.Error("Line with nothing but line break wasn't ignored")
cases := map[string]struct {
input string
want string
}{
"Line with nothing but line break": {
input: "\n",
},
"Line with nothing but windows-style line break": {
input: "\r\n",
},
"Line full of whitespace": {
input: "\t\t ",
},
"Comment": {
input: "# Comment",
},
"Indented comment": {
input: "\t # comment",
},
"non-ignored value": {
input: `export OPTION_B='\n'`,
want: `export OPTION_B='\n'`,
},
}
if !isIgnoredLine("\r\n") {
t.Error("Line with nothing but windows-style line break wasn't ignored")
}
if !isIgnoredLine("\t\t ") {
t.Error("Line full of whitespace wasn't ignored")
}
// it 'ignores comment lines' do
// expect(env("\n\n\n # HERE GOES FOO \nfoo=bar")).to eql('foo' => 'bar')
if !isIgnoredLine("# comment") {
t.Error("Comment wasn't ignored")
}
if !isIgnoredLine("\t#comment") {
t.Error("Indented comment wasn't ignored")
}
// make sure we're not getting false positives
if isIgnoredLine(`export OPTION_B='\n'`) {
t.Error("ignoring a perfectly valid line to parse")
for n, c := range cases {
t.Run(n, func(t *testing.T) {
got := string(getStatementStart([]byte(c.input)))
if got != c.want {
t.Errorf("Expected:\t %q\nGot:\t %q", c.want, got)
}
})
}
}
@ -424,6 +461,17 @@ func TestErrorParsing(t *testing.T) {
}
}
func TestComments(t *testing.T) {
envFileName := "fixtures/comments.env"
expectedValues := map[string]string{
"foo": "bar",
"bar": "foo#baz",
"baz": "foo",
}
loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets)
}
func TestWrite(t *testing.T) {
writeAndCompare := func(env string, expected string) {
envMap, _ := Unmarshal(env)

207
parser.go Normal file
View File

@ -0,0 +1,207 @@
package godotenv
import (
"bytes"
"errors"
"fmt"
"strings"
"unicode"
)
const (
charComment = '#'
prefixSingleQuote = '\''
prefixDoubleQuote = '"'
exportPrefix = "export"
)
func parseBytes(src []byte, out map[string]string) error {
src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1)
cutset := src
for {
cutset = getStatementStart(cutset)
if cutset == nil {
// reached end of file
break
}
key, left, err := locateKeyName(cutset)
if err != nil {
return err
}
value, left, err := extractVarValue(left, out)
if err != nil {
return err
}
out[key] = value
cutset = left
}
return nil
}
// getStatementPosition returns position of statement begin.
//
// It skips any comment line or non-whitespace character.
func getStatementStart(src []byte) []byte {
pos := indexOfNonSpaceChar(src)
if pos == -1 {
return nil
}
src = src[pos:]
if src[0] != charComment {
return src
}
// skip comment section
pos = bytes.IndexFunc(src, isCharFunc('\n'))
if pos == -1 {
return nil
}
return getStatementStart(src[pos:])
}
// locateKeyName locates and parses key name and returns rest of slice
func locateKeyName(src []byte) (key string, cutset []byte, err error) {
// trim "export" and space at beginning
src = bytes.TrimLeftFunc(bytes.TrimPrefix(src, []byte(exportPrefix)), isSpace)
// locate key name end and validate it in single loop
offset := 0
loop:
for i, char := range src {
rchar := rune(char)
if isSpace(rchar) {
continue
}
switch char {
case '=', ':':
// library also supports yaml-style value declaration
key = string(src[0:i])
offset = i + 1
break loop
case '_':
default:
// variable name should match [A-Za-z0-9_]
if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) {
continue
}
return "", nil, fmt.Errorf(
`unexpected character %q in variable name near %q`,
string(char), string(src))
}
}
if len(src) == 0 {
return "", nil, errors.New("zero length string")
}
// trim whitespace
key = strings.TrimRightFunc(key, unicode.IsSpace)
cutset = bytes.TrimLeftFunc(src[offset:], isSpace)
return key, cutset, nil
}
// extractVarValue extracts variable value and returns rest of slice
func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) {
quote, hasPrefix := hasQuotePrefix(src)
if !hasPrefix {
// unquoted value - read until whitespace
end := bytes.IndexFunc(src, unicode.IsSpace)
if end == -1 {
return expandVariables(string(src), vars), nil, nil
}
return expandVariables(string(src[0:end]), vars), src[end:], nil
}
// lookup quoted string terminator
for i := 1; i < len(src); i++ {
if char := src[i]; char != quote {
continue
}
// skip escaped quote symbol (\" or \', depends on quote)
if prevChar := src[i-1]; prevChar == '\\' {
continue
}
// trim quotes
trimFunc := isCharFunc(rune(quote))
value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc))
if quote == prefixDoubleQuote {
// unescape newlines for double quote (this is compat feature)
// and expand environment variables
value = expandVariables(expandEscapes(value), vars)
}
return value, src[i+1:], nil
}
// return formatted error if quoted string is not terminated
valEndIndex := bytes.IndexFunc(src, isCharFunc('\n'))
if valEndIndex == -1 {
valEndIndex = len(src)
}
return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex])
}
func expandEscapes(str string) string {
out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string {
c := strings.TrimPrefix(match, `\`)
switch c {
case "n":
return "\n"
case "r":
return "\r"
default:
return match
}
})
return unescapeCharsRegex.ReplaceAllString(out, "$1")
}
func indexOfNonSpaceChar(src []byte) int {
return bytes.IndexFunc(src, func(r rune) bool {
return !unicode.IsSpace(r)
})
}
// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character
func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) {
if len(src) == 0 {
return 0, false
}
switch prefix := src[0]; prefix {
case prefixDoubleQuote, prefixSingleQuote:
return prefix, true
default:
return 0, false
}
}
func isCharFunc(char rune) func(rune) bool {
return func(v rune) bool {
return v == char
}
}
// isSpace reports whether the rune is a space character but not line break character
//
// this differs from unicode.IsSpace, which also applies line break as space
func isSpace(r rune) bool {
switch r {
case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0:
return true
}
return false
}