diff --git a/.drone.yml b/.drone.yml index 2ccb54a..e59e626 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,8 +6,12 @@ steps: - name: simple tests image: golang:1.22.7 commands: - - go test ./... + - go test ./... - name: race tests image: golang:1.22.7 commands: - go test -race -count 100 -timeout 30s ./... + - name: benchmark + image: golang:1.22.7 + commands: + - go test -benchtime=1000x -benchmem -bench=Benchmark ./... | grep allocs diff --git a/Makefile b/Makefile index 4562697..4997fa7 100644 --- a/Makefile +++ b/Makefile @@ -10,5 +10,9 @@ test: race: go test -race -count 100 -timeout 30s ./... +.PHONY: bench +bench: ## Бенчмарки + go test -benchtime=1000x -benchmem -bench=Benchmark ./... | grep allocs + help: ## Print this help and exit @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/bytes2hex/main.go b/bytes_hex/main.go similarity index 100% rename from bytes2hex/main.go rename to bytes_hex/main.go diff --git a/bytes2string2bytes/main_test.go b/bytes_to_string_to_bytes/main_test.go similarity index 100% rename from bytes2string2bytes/main_test.go rename to bytes_to_string_to_bytes/main_test.go diff --git a/cutTheTree/main.go b/cutTheTree/main.go deleted file mode 100644 index d0ffde6..0000000 --- a/cutTheTree/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package cut_the_tree - -import ( - "encoding/json" - "maps" -) - -func (root Node) WeighingTreeWithRecursion() map[string]uint16 { - nodeBytes, _ := json.Marshal(Node{ - ID: root.ID, - Name: root.Name, - }) - weight := uint16(len(nodeBytes)) - var res = make(map[string]uint16) - for _, v := range root.Children { - maps.Copy(res, v.WeighingTreeWithRecursion()) // Т.е. в go нет оптимизации хвостовой рекурсии, это породит неимоверное кол-во аллокаций. - weight = weight + res[v.ID] - } - res[root.ID] = weight - return res -} - -func (root Node) WeighingTreeIdiomaticDFS() map[string]uint16 { - return nil -} - -func (root Node) WeighingTreeIdiomaticBFS() map[string]uint16 { - return nil -} - -func (root Node) DecomposeTree(weights map[string]uint16, limit int) ([]*Node, error) { - - return nil, nil -} diff --git a/cutTheTree/models.go b/cutTheTree/models.go deleted file mode 100644 index 21837a3..0000000 --- a/cutTheTree/models.go +++ /dev/null @@ -1,7 +0,0 @@ -package cut_the_tree - -type Node struct { - ID string `json:"id"` - Name string `json:"name"` - Children []*Node `json:"children"` -} diff --git a/cutTheTree/README.md b/cut_the_tree/README.md similarity index 100% rename from cutTheTree/README.md rename to cut_the_tree/README.md diff --git a/cut_the_tree/funcs.go b/cut_the_tree/funcs.go new file mode 100644 index 0000000..54dc863 --- /dev/null +++ b/cut_the_tree/funcs.go @@ -0,0 +1,84 @@ +package cut_the_tree + +import ( + "encoding/json" + "maps" +) + +type Node struct { + ID string `json:"id"` + Name string `json:"name"` + Children []*Node `json:"children"` +} + +type WeightedNode struct { + parent string + weight uint16 +} + +func (root Node) WeighingTreeWithRecursion() map[string]uint16 { + weight := getNodeWeight(root.ID, root.Name) + var res = make(map[string]uint16) + for _, v := range root.Children { + maps.Copy(res, v.WeighingTreeWithRecursion()) // Т.е. в go нет оптимизации хвостовой рекурсии, это породит неимоверное кол-во аллокаций. + weight = weight + res[v.ID] + } + res[root.ID] = weight + return res +} + +func (root Node) WeighingTreeIdiomaticBFS() map[string]uint16 { + weight := getNodeWeight(root.ID, root.Name) + counter := map[string]*WeightedNode{root.ID: { + weight: weight, + }} + stack := []*Node{&root} + for len(stack) > 0 { + current := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + nodeWeight := getNodeWeight(current.ID, current.Name) + counter[current.ID].weight = nodeWeight + // Прибавляем вес всем родительским нодам до корня + parentID := counter[current.ID].parent + for { + currentNode := counter[parentID] + if parentID == "" { + break + } + currentNode.weight = currentNode.weight + nodeWeight + parentID = currentNode.parent + } + + for _, child := range current.Children { + counter[child.ID] = &WeightedNode{ + parent: current.ID, + } + stack = append(stack, child) + } + } + + var res = make(map[string]uint16) + for k, v := range counter { + res[k] = v.weight + } + + return res +} + +func (root Node) WeighingTreeIdiomaticDFS() map[string]uint16 { + return nil +} + +func (root Node) DecomposeTree(weights map[string]uint16, limit int) ([]*Node, error) { + + return nil, nil +} + +func getNodeWeight(id string, name string) uint16 { + nodeBytes, _ := json.Marshal(Node{ + ID: id, + Name: name, + }) + return uint16(len(nodeBytes)) +} diff --git a/cutTheTree/main_bench_test.go b/cut_the_tree/funcs_bench_test.go similarity index 53% rename from cutTheTree/main_bench_test.go rename to cut_the_tree/funcs_bench_test.go index 531a6cf..999820c 100644 --- a/cutTheTree/main_bench_test.go +++ b/cut_the_tree/funcs_bench_test.go @@ -13,3 +13,12 @@ func BenchmarkNode_WeighingTreeWithRecursion(b *testing.B) { assert.EqualValues(b, output, testOrdinaryVFS.WeighingTreeWithRecursion()) } } + + +func BenchmarkNode_WeighingTreeIdiomaticBFS(b *testing.B) { + for i := 0; i < b.N; i++ { + output := map[string]uint16{"node0":1416, "node01":194, "node011":49, "node012":49, "node013":49, "node02":194, "node021":49, "node022":49, "node023":49, "node03":983,"node031":361, "node0311":210, "node03111":53, "node03112":53, "node03113":53, "node0312":51, "node0313":51, "node032":49, "node033":526, "node0331":51, "node0332":375, "node03321":53, "node03322":53, "node03323":218, "node033231":55, "node033232":55, "node033233":55, "node0333":51} + + assert.EqualValues(b, output, testOrdinaryVFS.WeighingTreeIdiomaticBFS()) + } +} diff --git a/cutTheTree/main_test.go b/cut_the_tree/funcs_models_test.go similarity index 76% rename from cutTheTree/main_test.go rename to cut_the_tree/funcs_models_test.go index b0cad1a..199a748 100644 --- a/cutTheTree/main_test.go +++ b/cut_the_tree/funcs_models_test.go @@ -1,38 +1,7 @@ package cut_the_tree_test -import ( - "github.com/stretchr/testify/assert" -"testing" +import "tests/cut_the_tree" - cut_the_tree "tests/cutTheTree" - - ) - -func TestNode_WeighingTreeAllAlgo(t *testing.T) { - for name,tt := range map[string]struct{ - input cut_the_tree.Node - output map[string]uint16 - }{ - "Ordinary VFS tree":{ - input:testOrdinaryVFS, - output: map[string]uint16{"node0":1416, "node01":194, "node011":49, "node012":49, "node013":49, "node02":194, "node021":49, "node022":49, "node023":49, "node03":983,"node031":361, "node0311":210, "node03111":53, "node03112":53, "node03113":53, "node0312":51, "node0313":51, "node032":49, "node033":526, "node0331":51, "node0332":375, "node03321":53, "node03322":53, "node03323":218, "node033231":55, "node033232":55, "node033233":55, "node0333":51}, - }, - "VFS in the form of a pathological tree":{ - input: testPathologicalTree, - output: map[string]uint16{"node0":605, "node01":560, "node012":513, "node0123":464, "node01234":413, "node012345":360, "node0123456":305, "node01234567":248, "node012345678":189, "node0123456789":128, "node0123456789A":65}, - }, - }{ - t.Run(name,func(t *testing.T) { - assert.EqualValues(t, tt.output, tt.input.WeighingTreeWithRecursion()) - assert.EqualValues(t, tt.output, tt.input.WeighingTreeIdiomaticDFS()) - assert.EqualValues(t, tt.output, tt.input.WeighingTreeIdiomaticBFS()) - }) - } -} - -func TestNode_DecomposeTree(t *testing.T) { - -} var testOrdinaryVFS = cut_the_tree.Node{ ID: "node0", diff --git a/cut_the_tree/funcs_test.go b/cut_the_tree/funcs_test.go new file mode 100644 index 0000000..0f5dc16 --- /dev/null +++ b/cut_the_tree/funcs_test.go @@ -0,0 +1,35 @@ +package cut_the_tree_test + +import ( + "github.com/stretchr/testify/assert" +"testing" + + cut_the_tree "tests/cut_the_tree" + + ) + +func TestNode_WeighingTreeAllAlgo(t *testing.T) { + for name,tt := range map[string]struct{ + input cut_the_tree.Node + output map[string]uint16 + }{ + "Ordinary VFS tree":{ + input:testOrdinaryVFS, + output: map[string]uint16{"node0":1416, "node01":194, "node011":49, "node012":49, "node013":49, "node02":194, "node021":49, "node022":49, "node023":49, "node03":983,"node031":361, "node0311":210, "node03111":53, "node03112":53, "node03113":53, "node0312":51, "node0313":51, "node032":49, "node033":526, "node0331":51, "node0332":375, "node03321":53, "node03322":53, "node03323":218, "node033231":55, "node033232":55, "node033233":55, "node0333":51}, + }, + "VFS in the form of a pathological tree":{ + input: testPathologicalTree, + output: map[string]uint16{"node0":605, "node01":560, "node012":513, "node0123":464, "node01234":413, "node012345":360, "node0123456":305, "node01234567":248, "node012345678":189, "node0123456789":128, "node0123456789A":65}, + }, + }{ + t.Run(name,func(t *testing.T) { + assert.EqualValues(t, tt.output, tt.input.WeighingTreeWithRecursion()) + assert.EqualValues(t, tt.output, tt.input.WeighingTreeIdiomaticBFS()) + assert.EqualValues(t, tt.output, tt.input.WeighingTreeIdiomaticDFS()) + }) + } +} + +func TestNode_DecomposeTree(t *testing.T) { + +} diff --git a/file_checksum/file_hash.go b/file_checksum/file_hash.go new file mode 100644 index 0000000..d9147de --- /dev/null +++ b/file_checksum/file_hash.go @@ -0,0 +1,64 @@ +// Package checksum computes checksums, like MD5 or SHA256, for large files +package main + +import ( + "bufio" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "hash" + "os" + + "golang.org/x/crypto/blake2s" +) + +// MD5sum returns MD5 checksum of filename +func MD5sum(filename string) (string, error) { + return sum(md5.New(), filename) +} + +// SHA256sum returns SHA256 checksum of filename +func SHA256sum(filename string) (string, error) { + return sum(sha256.New(), filename) +} + +// SHA1sum returns SHA1 checksum of filename +func SHA1sum(filename string) (string, error) { + return sum(sha1.New(), filename) +} + +// Blake2s256 returns BLAKE2s-256 checksum of filename +func Blake2s256(filename string) (string, error) { + hash, _ := blake2s.New256([]byte{}) + return sum(hash, filename) +} + +// CRC32 returns CRC-32-IEEE checksum of filename +func CRC32(filename string) (string, error) { + if info, err := os.Stat(filename); err != nil || info.IsDir() { + return "", err + } + + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + + return CRCReader(bufio.NewReader(file)) +} + +// sum calculates the hash based on a provided hash provider +func sum(hashAlgorithm hash.Hash, filename string) (string, error) { + if info, err := os.Stat(filename); err != nil || info.IsDir() { + return "", err + } + + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + + return sumReader(hashAlgorithm, bufio.NewReader(file)) +} diff --git a/file_checksum/file_hash_test.go b/file_checksum/file_hash_test.go new file mode 100644 index 0000000..ef07d98 --- /dev/null +++ b/file_checksum/file_hash_test.go @@ -0,0 +1,89 @@ +package main_test + +import ( + "fmt" + "github.com/codingsince1985/checksum" + "os" + "testing" +) + +func prepareFile() (string, error) { + file, err := os.CreateTemp("", "prefix") + if err != nil { + return "", err + } + if err := os.WriteFile(file.Name(), []byte("some data"), 0600); err != nil { + return "", err + } + return file.Name(), nil +} + +func testFile(t *testing.T, checksumFunc func(string) error) { + file, err := prepareFile() + if err != nil { + t.Logf("could not create test file: %s", err) + t.FailNow() + } + defer func() { + err := os.Remove(file) + if err != nil { + t.Logf("could not remove test file: %s", err) + } + }() + if err := checksumFunc(file); err != nil { + t.Error(err) + } +} + +func TestBlake2s256(t *testing.T) { + testFile(t, func(filename string) error { + if result, err := checksum.Blake2s256(filename); err != nil || result != "54fc4fe89148c8f82479348f56168f71c4165eedda67961daec1d46015db3884" { + return fmt.Errorf(result, err) + } + return nil + }) +} + +func TestSHA1sumFile(t *testing.T) { + testFile(t, func(filename string) error { + if result, err := checksum.SHA1sum(filename); err != nil || result != "baf34551fecb48acc3da868eb85e1b6dac9de356" { + return fmt.Errorf(result, err) + } + return nil + }) +} + +func TestSHA256sumFile(t *testing.T) { + testFile(t, func(filename string) error { + if result, err := checksum.SHA256sum(filename); err != nil || result != "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee" { + return fmt.Errorf(result, err) + } + return nil + }) +} + +func TestMd5sumFile(t *testing.T) { + testFile(t, func(filename string) error { + if result, err := checksum.MD5sum(filename); err != nil || result != "1e50210a0202497fb79bc38b6ade6c34" { + return fmt.Errorf(result, err) + } + return nil + }) +} + +func TestCrc32File(t *testing.T) { + testFile(t, func(filename string) error { + if result, err := checksum.CRC32(filename); err != nil || result != "d9c2e91e" { + return fmt.Errorf(result, err) + } + return nil + }) +} + +func TestMd5sumDir(t *testing.T) { + os.MkdirAll("/tmp/downloads", 777) + defer os.Remove("/tmp/downloads") + if md5sum, err := checksum.MD5sum("/tmp/downloads"); err != nil || md5sum != "" { + t.Error("Md5sum(dir) failed", md5sum, err) + } +} diff --git a/file_checksum/hash.go b/file_checksum/hash.go new file mode 100644 index 0000000..7574f12 --- /dev/null +++ b/file_checksum/hash.go @@ -0,0 +1,68 @@ +package main + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "fmt" + "hash" + "hash/crc32" + "io" + + "golang.org/x/crypto/blake2s" +) + +const bufferSize = 65536 + +// MD5sumReader returns MD5 checksum of content in reader +func MD5sumReader(reader io.Reader) (string, error) { + return sumReader(md5.New(), reader) +} + +// SHA256sumReader returns SHA256 checksum of content in reader +func SHA256sumReader(reader io.Reader) (string, error) { + return sumReader(sha256.New(), reader) +} + +// SHA1sumReader returns SHA1 checksum of content in reader +func SHA1sumReader(reader io.Reader) (string, error) { + return sumReader(sha1.New(), reader) +} + +// Blake2s256Reader returns SHA1 checksum of content in reader +func Blake2s256Reader(reader io.Reader) (string, error) { + hash, _ := blake2s.New256([]byte{}) + return sumReader(hash, reader) +} + +// CRCReader returns CRC-32-IEEE checksum of content in reader +func CRCReader(reader io.Reader) (string, error) { + table := crc32.MakeTable(crc32.IEEE) + checksum := crc32.Checksum([]byte(""), table) + buf := make([]byte, bufferSize) + for { + switch n, err := reader.Read(buf); err { + case nil: + checksum = crc32.Update(checksum, table, buf[:n]) + case io.EOF: + return fmt.Sprintf("%08x", checksum), nil + default: + return "", err + } + } +} + +// sumReader calculates the hash based on a provided hash provider +func sumReader(hashAlgorithm hash.Hash, reader io.Reader) (string, error) { + buf := make([]byte, bufferSize) + for { + switch n, err := reader.Read(buf); err { + case nil: + hashAlgorithm.Write(buf[:n]) + case io.EOF: + return fmt.Sprintf("%x", hashAlgorithm.Sum(nil)), nil + default: + return "", err + } + } +} diff --git a/file_checksum/hash_test.go b/file_checksum/hash_test.go new file mode 100644 index 0000000..4b76c23 --- /dev/null +++ b/file_checksum/hash_test.go @@ -0,0 +1,37 @@ +package main_test + +import ( + "github.com/codingsince1985/checksum" + "strings" + "testing" +) + +func TestSHA1sumReader(t *testing.T) { + if result, err := checksum.SHA1sumReader(strings.NewReader("some data")); err != nil || result != "baf34551fecb48acc3da868eb85e1b6dac9de356" { + t.Error(result, err) + } +} + +func TestSHA256sumReader(t *testing.T) { + if result, err := checksum.SHA256sumReader(strings.NewReader("dot")); err != nil || result != "e392dad8b08599f74d4819cd291feef81ab4389e0a6fae2b1286f99411b0c7ca" { + t.Error(result, err) + } +} + +func TestMd5sumReader(t *testing.T) { + if result, err := checksum.MD5sumReader(strings.NewReader("dot")); err != nil || result != "69eb76c88557a8211cbfc9beda5fc062" { + t.Error(result, err) + } +} + +func TestCrcReader(t *testing.T) { + if result, err := checksum.CRCReader(strings.NewReader("dot")); err != nil || result != "059278a3" { + t.Error(result, err) + } +} + +func TestBlake2s256Reader(t *testing.T) { + if result, err := checksum.Blake2s256Reader(strings.NewReader("dot")); err != nil || result != "e364d604d2573afa9f0882f8a50458c4c9c16ca185ab97c5ec1fc71bfc7063bf" { + t.Error(result, err) + } +} diff --git a/go.mod b/go.mod index 8c65fab..4a9bfde 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module tests go 1.21 require ( - github.com/jsimonetti/berkeleydb v0.0.0-20170815141343-5cde5eaaf78c + github.com/codingsince1985/checksum v1.3.0 + github.com/mitchellh/go-homedir v1.1.0 github.com/stretchr/testify v1.9.0 + golang.org/x/crypto v0.27.0 ) require ( @@ -12,6 +14,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.25.0 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/runonce/cleaner/singleton.go b/run_once/cleaner/singleton.go similarity index 100% rename from runonce/cleaner/singleton.go rename to run_once/cleaner/singleton.go diff --git a/runonce/main.go b/run_once/main.go similarity index 96% rename from runonce/main.go rename to run_once/main.go index 0ddf7aa..fdc648f 100644 --- a/runonce/main.go +++ b/run_once/main.go @@ -4,7 +4,7 @@ import ( "fmt" "log" "sync" - "tests/runonce/cleaner" + "tests/run_once/cleaner" ) type srv struct {