From bef1c47dfd5d07918bc9545a837c4c7e8d62c834 Mon Sep 17 00:00:00 2001 From: AL <75311052+AAVision@users.noreply.github.com> Date: Tue, 1 Apr 2025 01:27:24 +0300 Subject: [PATCH 1/3] Adding new hashing conditions --- go.mod | 3 +++ go.sum | 6 +++++ hash.go | 67 +++++++++++++++++++++++++++++++++++++++++++------ hash_test.go | 3 ++- main.go | 3 ++- process.go | 31 +++++++++++++++++------ process_test.go | 3 ++- 7 files changed, 99 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index f7fa22e..33066fd 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,11 @@ require ( require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/x/ansi v0.8.0 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -21,6 +23,7 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect diff --git a/go.sum b/go.sum index 54e585a..3c8ac9b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI= github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= @@ -10,6 +12,8 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -27,6 +31,8 @@ github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1n github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= diff --git a/hash.go b/hash.go index 9f88f3a..2a5e9f5 100644 --- a/hash.go +++ b/hash.go @@ -1,29 +1,82 @@ package main import ( + "context" "encoding/hex" "fmt" "io" "os" + "sync" + "github.com/zeebo/xxh3" "golang.org/x/crypto/blake2b" ) -func createFileHash(filePath string) (string, error) { +const ( + hashThresholdMB = 20 // Files larger than this will use BLAKE2b + bufferSize = 32 * 1024 +) + +var bufPool = sync.Pool{ + New: func() interface{}{return make([]byte, bufferSize)}, +} + +func createFileHash(ctx context.Context, filePath string) (string, int64, error){ file, err := os.Open(filePath) if err != nil { - return "", fmt.Errorf("failed to open file: %w", err) + return "", 0, fmt.Errorf("failed to open file: %w", err) } defer file.Close() - hash, err := blake2b.New256(nil) + fileInfo, err := file.Stat() + if err != nil { + return "", 0, fmt.Errorf("failed to get file stats: %w", err) + } + + fileSize := fileInfo.Size() + + sizeMB := float64(fileSize) / (1024 * 1024) + var hash string + + if sizeMB > hashThresholdMB{ + hash, err = hashBlake2b(file) + } else{ + hash, err = hashXXH3(file) + } + if err != nil { - return "", fmt.Errorf("failed to create BLAKE2b hash: %w", err) + return "", fileSize, fmt.Errorf("hashing failed: %w", err) } - if _, err := io.Copy(hash, file); err != nil { - return "", fmt.Errorf("failed to compute hash: %w", err) + return hash, fileSize, nil + +} + +func hashBlake2b(r io.Reader) (string, error){ + hasher, err := blake2b.New256(nil) + if err != nil { + return "", fmt.Errorf("failed to create BLAKE2b hasher: %w", err) } - return hex.EncodeToString(hash.Sum(nil)), nil + buff := bufPool.Get().([]byte) + defer bufPool.Put(buff) + + if _, err := io.CopyBuffer(hasher, r, buff); err != nil { + return "", fmt.Errorf("BLAKE2b hashing failed: %w", err) + } + + return hex.EncodeToString(hasher.Sum(nil)), nil } + +func hashXXH3(r io.Reader) (string, error) { + hasher := xxh3.New() + buff := bufPool.Get().([]byte) + defer bufPool.Put(buff) + + if _, err := io.CopyBuffer(hasher, r, buff); err != nil { + return "", fmt.Errorf("XXH3 hashing failed: %w", err) + } + + hash := hasher.Sum128() + return fmt.Sprintf("%016x%016x", hash.Hi, hash.Lo), nil +} \ No newline at end of file diff --git a/hash_test.go b/hash_test.go index 81c31f6..1106db5 100644 --- a/hash_test.go +++ b/hash_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/hex" "io" "os" @@ -84,7 +85,7 @@ func TestCreateFileHash(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - hash, err := createFileHash(tc.filePath) + hash, _, err := createFileHash(context.Background(), tc.filePath) if tc.expectedErrMsg != "" { if err == nil { diff --git a/main.go b/main.go index 63eb991..7acd089 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -69,7 +70,7 @@ func main() { sem <- struct{}{} defer func() { <-sem }() - res, err := groupByHash(files, workers) + res, err := groupByHash(context.Background(), files, workers) if err != nil { log.Printf("Error: %v", err) return diff --git a/process.go b/process.go index 0e21c7a..ca35bb1 100644 --- a/process.go +++ b/process.go @@ -2,39 +2,55 @@ package main import ( "bytes" + "context" "fmt" "io" "os" "sync" ) -func groupByHash(files []string, workers uint) (map[string][]string, error) { +func groupByHash(ctx context.Context, files []string, workers uint) (map[string][]string, error) { type hashResult struct { hash string file string err error } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + hashChan := make(chan string, len(files)) resultChan := make(chan hashResult, len(files)) var wg sync.WaitGroup - for range workers { - wg.Add(1) + + wg.Add(int(workers)) + + for i := 0; i < int(workers); i++ { go func() { defer wg.Done() for file := range hashChan { - h, err := createFileHash(file) - resultChan <- hashResult{h, file, err} + select { + case <-ctx.Done(): + return + default: + hash, _, err := createFileHash(ctx, file) + resultChan <- hashResult{hash, file, err} + } } }() } go func() { + defer close(hashChan) + for _, file := range files { - hashChan <- file + select{ + case hashChan <- file: + case <-ctx.Done(): + return + } } - close(hashChan) }() go func() { @@ -45,6 +61,7 @@ func groupByHash(files []string, workers uint) (map[string][]string, error) { m := make(map[string][]string) for res := range resultChan { if res.err != nil { + cancel() return nil, res.err } m[res.hash] = append(m[res.hash], res.file) diff --git a/process_test.go b/process_test.go index 17be338..5cc967a 100644 --- a/process_test.go +++ b/process_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/md5" "encoding/hex" "io" @@ -58,7 +59,7 @@ func TestGroupByHash(t *testing.T) { filesNames = append(filesNames, filepath.Join(tempDir, v.file)) } - m, err := groupByHash(filesNames, workers) + m, err := groupByHash(context.Background(), filesNames, workers) if err != nil { t.Error(err) } From bece04ca166ec7bcd89f12c0a6625718c79bd9b0 Mon Sep 17 00:00:00 2001 From: AL <75311052+AAVision@users.noreply.github.com> Date: Wed, 2 Apr 2025 17:30:22 +0300 Subject: [PATCH 2/3] keeping only the XXH3 --- go.mod | 4 +--- go.sum | 6 ++---- hash.go | 37 +++++++++---------------------------- hash_test.go | 19 ++++++++++--------- 4 files changed, 22 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 33066fd..da20cbc 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,11 @@ go 1.24.0 require ( github.com/charmbracelet/bubbletea v1.3.4 github.com/charmbracelet/lipgloss v1.0.0 - golang.org/x/crypto v0.36.0 + github.com/zeebo/xxh3 v1.0.2 ) require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/x/ansi v0.8.0 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect @@ -23,7 +22,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect diff --git a/go.sum b/go.sum index 3c8ac9b..c821438 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI= github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= @@ -31,10 +29,10 @@ github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1n github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/hash.go b/hash.go index 2a5e9f5..4864556 100644 --- a/hash.go +++ b/hash.go @@ -2,23 +2,23 @@ package main import ( "context" - "encoding/hex" "fmt" "io" "os" "sync" "github.com/zeebo/xxh3" - "golang.org/x/crypto/blake2b" ) const ( - hashThresholdMB = 20 // Files larger than this will use BLAKE2b bufferSize = 32 * 1024 ) var bufPool = sync.Pool{ - New: func() interface{}{return make([]byte, bufferSize)}, + New: func() interface{} { + b := make([]byte, bufferSize) + return &b + }, } func createFileHash(ctx context.Context, filePath string) (string, int64, error){ @@ -35,14 +35,10 @@ func createFileHash(ctx context.Context, filePath string) (string, int64, error) fileSize := fileInfo.Size() - sizeMB := float64(fileSize) / (1024 * 1024) var hash string - if sizeMB > hashThresholdMB{ - hash, err = hashBlake2b(file) - } else{ - hash, err = hashXXH3(file) - } + hash, err = hashXXH3(file) + if err != nil { return "", fileSize, fmt.Errorf("hashing failed: %w", err) @@ -52,26 +48,11 @@ func createFileHash(ctx context.Context, filePath string) (string, int64, error) } -func hashBlake2b(r io.Reader) (string, error){ - hasher, err := blake2b.New256(nil) - if err != nil { - return "", fmt.Errorf("failed to create BLAKE2b hasher: %w", err) - } - - buff := bufPool.Get().([]byte) - defer bufPool.Put(buff) - - if _, err := io.CopyBuffer(hasher, r, buff); err != nil { - return "", fmt.Errorf("BLAKE2b hashing failed: %w", err) - } - - return hex.EncodeToString(hasher.Sum(nil)), nil -} - func hashXXH3(r io.Reader) (string, error) { hasher := xxh3.New() - buff := bufPool.Get().([]byte) - defer bufPool.Put(buff) + buffPtr := bufPool.Get().(*[]byte) + buff := *buffPtr + defer bufPool.Put(buffPtr) if _, err := io.CopyBuffer(hasher, r, buff); err != nil { return "", fmt.Errorf("XXH3 hashing failed: %w", err) diff --git a/hash_test.go b/hash_test.go index 1106db5..75d6dd5 100644 --- a/hash_test.go +++ b/hash_test.go @@ -3,12 +3,13 @@ package main import ( "context" "encoding/hex" + "fmt" "io" "os" "path/filepath" "testing" - "golang.org/x/crypto/blake2b" + "github.com/zeebo/xxh3" ) func TestCreateFileHash(t *testing.T) { @@ -32,22 +33,22 @@ func TestCreateFileHash(t *testing.T) { } contentFile.Close() - h, err := blake2b.New256(nil) - if err != nil { - t.Fatalf("Failed to create BLAKE2b hash: %v", err) - } - n, err := io.WriteString(h, content) + hasher := xxh3.New() + + n, err := io.WriteString(hasher, content) if err != nil { t.Error(err) } if n != len(content) { t.Errorf("not all bytes are written, expected to write %d bytes, written: %d", len(content), n) } - expectedContentHash := hex.EncodeToString(h.Sum(nil)) - emptyHash, err := blake2b.New256(nil) + hash := hasher.Sum128() + expectedContentHash := fmt.Sprintf("%016x%016x", hash.Hi, hash.Lo) + + emptyHash := xxh3.New() if err != nil { - t.Fatalf("Failed to create BLAKE2b hash for empty file: %v", err) + t.Fatalf("Failed to create XXH3 hash for empty file: %v", err) } expectedEmptyHash := hex.EncodeToString(emptyHash.Sum(nil)) From cc66d0bd06eb4bb67aed02c89f56fff13d352789 Mon Sep 17 00:00:00 2001 From: AL <75311052+AAVision@users.noreply.github.com> Date: Thu, 10 Apr 2025 08:53:41 +0300 Subject: [PATCH 3/3] Removing context --- hash.go | 21 ++++++--------------- hash_test.go | 5 ++--- main.go | 3 +-- process.go | 31 +++++++------------------------ process_test.go | 3 +-- 5 files changed, 17 insertions(+), 46 deletions(-) diff --git a/hash.go b/hash.go index 4864556..7fe58f9 100644 --- a/hash.go +++ b/hash.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "io" "os" @@ -11,7 +10,7 @@ import ( ) const ( - bufferSize = 32 * 1024 + bufferSize = 32 * 1024 ) var bufPool = sync.Pool{ @@ -21,30 +20,22 @@ var bufPool = sync.Pool{ }, } -func createFileHash(ctx context.Context, filePath string) (string, int64, error){ +func createFileHash(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { - return "", 0, fmt.Errorf("failed to open file: %w", err) + return "", fmt.Errorf("failed to open file: %w", err) } defer file.Close() - fileInfo, err := file.Stat() - if err != nil { - return "", 0, fmt.Errorf("failed to get file stats: %w", err) - } - - fileSize := fileInfo.Size() - var hash string hash, err = hashXXH3(file) - if err != nil { - return "", fileSize, fmt.Errorf("hashing failed: %w", err) + return "", fmt.Errorf("hashing failed: %w", err) } - return hash, fileSize, nil + return hash, nil } @@ -60,4 +51,4 @@ func hashXXH3(r io.Reader) (string, error) { hash := hasher.Sum128() return fmt.Sprintf("%016x%016x", hash.Hi, hash.Lo), nil -} \ No newline at end of file +} diff --git a/hash_test.go b/hash_test.go index 75d6dd5..36e731c 100644 --- a/hash_test.go +++ b/hash_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/hex" "fmt" "io" @@ -34,7 +33,7 @@ func TestCreateFileHash(t *testing.T) { contentFile.Close() hasher := xxh3.New() - + n, err := io.WriteString(hasher, content) if err != nil { t.Error(err) @@ -86,7 +85,7 @@ func TestCreateFileHash(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - hash, _, err := createFileHash(context.Background(), tc.filePath) + hash, err := createFileHash(tc.filePath) if tc.expectedErrMsg != "" { if err == nil { diff --git a/main.go b/main.go index 7acd089..63eb991 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "flag" "fmt" "log" @@ -70,7 +69,7 @@ func main() { sem <- struct{}{} defer func() { <-sem }() - res, err := groupByHash(context.Background(), files, workers) + res, err := groupByHash(files, workers) if err != nil { log.Printf("Error: %v", err) return diff --git a/process.go b/process.go index ca35bb1..0e21c7a 100644 --- a/process.go +++ b/process.go @@ -2,55 +2,39 @@ package main import ( "bytes" - "context" "fmt" "io" "os" "sync" ) -func groupByHash(ctx context.Context, files []string, workers uint) (map[string][]string, error) { +func groupByHash(files []string, workers uint) (map[string][]string, error) { type hashResult struct { hash string file string err error } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - hashChan := make(chan string, len(files)) resultChan := make(chan hashResult, len(files)) var wg sync.WaitGroup - - wg.Add(int(workers)) - - for i := 0; i < int(workers); i++ { + for range workers { + wg.Add(1) go func() { defer wg.Done() for file := range hashChan { - select { - case <-ctx.Done(): - return - default: - hash, _, err := createFileHash(ctx, file) - resultChan <- hashResult{hash, file, err} - } + h, err := createFileHash(file) + resultChan <- hashResult{h, file, err} } }() } go func() { - defer close(hashChan) - for _, file := range files { - select{ - case hashChan <- file: - case <-ctx.Done(): - return - } + hashChan <- file } + close(hashChan) }() go func() { @@ -61,7 +45,6 @@ func groupByHash(ctx context.Context, files []string, workers uint) (map[string] m := make(map[string][]string) for res := range resultChan { if res.err != nil { - cancel() return nil, res.err } m[res.hash] = append(m[res.hash], res.file) diff --git a/process_test.go b/process_test.go index 5cc967a..17be338 100644 --- a/process_test.go +++ b/process_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "crypto/md5" "encoding/hex" "io" @@ -59,7 +58,7 @@ func TestGroupByHash(t *testing.T) { filesNames = append(filesNames, filepath.Join(tempDir, v.file)) } - m, err := groupByHash(context.Background(), filesNames, workers) + m, err := groupByHash(filesNames, workers) if err != nil { t.Error(err) }