Increase speed and efficiency of operations

.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Run tests
+        run: |
+          go test ./... -v -coverprofile=coverage.out
+
+      - name: Run vet
+        run: |
+          go vet ./...
+
+      - name: Upload coverage
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.out
+
+  build:
+    name: Build
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin]
+        goarch: [amd64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+
+      - name: Build binary
+        run: |
+          mkdir -p build
+          BIN_NAME=mediarizer2
+          if [ "${{ matrix.goos }}" = "windows" ]; then BIN_NAME=mediarizer2.exe; fi
+          env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o build/${BIN_NAME}-${{ matrix.goos }}-${{ matrix.goarch }} ./app
+
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: mediarizer2-${{ matrix.goos }}-${{ matrix.goarch }}
+          path: build/

app/consumer.go

@@ -19,12 +19,15 @@ func consumer(
 	geoLocation bool,
 	format string,
 	verbose bool,
-	duplicateStrategy string,
 	processedFiles *int64,
-	done chan<- struct{}) {
+	done chan<- struct{},
+	dupChecker *duplicate.DuplicateChecker) {
 
 	var wg sync.WaitGroup
 	numWorkers := runtime.NumCPU() / 2
+	if numWorkers < 1 {
+		numWorkers = 1
+	}
 
 	for i := 0; i < numWorkers; i++ {
 		wg.Add(1)
@@ -38,7 +41,7 @@ func consumer(
 					geoLocation,
 					format,
 					verbose,
-					duplicateStrategy,
+					dupChecker,
 				)
 
 				atomic.AddInt64(processedFiles, 1)
@@ -57,55 +60,64 @@ func processFileInfo(
 	geoLocation bool,
 	format string,
 	verbose bool,
-	duplicateStrategy string,
+	dupChecker *duplicate.DuplicateChecker,
 ) {
-	var generatedPath string
-	var err error
+	// Check for duplicates first
+	isDup, originalPath, err := dupChecker.CheckAndTrack(fileInfo.Path)
+	if err != nil {
+		errorQueue <- fmt.Errorf("failed to check duplicate for %s: %v", fileInfo.Path, err)
+		return
+	}
+
+	if isDup {
+		duplicatesDir := filepath.Join(destinationPath, "duplicates")
+		if err := duplicate.MoveDuplicate(fileInfo.Path, originalPath, duplicatesDir); err != nil {
+			errorQueue <- fmt.Errorf("failed to move duplicate %s: %v", fileInfo.Path, err)
+		}
+		return
+	}
 
+	var generatedPath string
 	generatedPath, err = getDestinationPath(destinationPath, fileInfo, geoLocation, format)
 	if err != nil {
 		errorQueue <- err
 		return
 	}
 
-	if fileInfo.isDuplicate {
-		generatedPath, err = duplicate.CreateDuplicateFolder(generatedPath, "DUPLICATE")
+	// Check if file already exists and add numeric suffix if needed
+	_, err = os.Stat(generatedPath)
+	if !os.IsNotExist(err) {
+		generatedPath, err = generateUniquePathName(generatedPath)
 		if err != nil {
 			errorQueue <- err
 			return
 		}
-		generatedPath = filepath.Join(generatedPath, filepath.Base(fileInfo.Path))
-	} else {
-		_, err = os.Stat(generatedPath)
-		if !os.IsNotExist(err) {
-			generatedPath, err = generateUniquePathName(generatedPath)
-			if err != nil {
-				errorQueue <- err
-				return
-			}
-		}
 	}
 
 	err = moveFile(
 		fileInfo.Path,
 		generatedPath,
 		verbose,
-		fileInfo.isDuplicate,
-		duplicateStrategy,
 	)
 	if err != nil {
 		errorQueue <- fmt.Errorf("failed to move %s to %s: %v", fileInfo.Path, generatedPath, err)
+	} else {
+		// Track the new file in the index
+		if err := dupChecker.TrackNewFile(generatedPath); err != nil {
+			// Log warning but don't fail
+			// fmt.Printf("Warning: failed to track new file %s: %v\n", generatedPath, err)
+		}
 	}
 }
 
-func moveFile(sourcePath, destinationPath string, verbose bool, isDuplicate bool, duplicateStrategy string) error {
+func moveFile(sourcePath, destinationPath string, verbose bool) error {
 	destPath := filepath.Dir(destinationPath)
 	if err := os.MkdirAll(destPath, os.ModePerm); err != nil {
 		return fmt.Errorf("failed to create destination directory %s: %v", destPath, err)
 	}
 
 	if verbose {
-		moveActionLog, err := logMoveAction(sourcePath, destPath, isDuplicate, duplicateStrategy)
+		moveActionLog, err := logMoveAction(sourcePath, destPath)
 		if err != nil {
 			return err
 		}

app/consumer_test.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGenerateUniquePathName_AppendsCounter(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "file.jpg")
+	if err := os.WriteFile(p, []byte("x"), 0644); err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	p2, err := generateUniquePathName(p)
+	if err != nil {
+		t.Fatalf("generateUniquePathName: %v", err)
+	}
+	if p2 == p {
+		t.Fatalf("expected different path")
+	}
+	if filepath.Ext(p2) != ".jpg" {
+		t.Fatalf("expected same extension")
+	}
+}

app/creator.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/keybraker/mediarizer-2/duplicate"
-
 	"github.com/rwcarlsen/goexif/exif"
 )
 
@@ -27,7 +25,6 @@ func creator(
 	fileTypesToInclude []string,
 	organisePhotos bool,
 	organiseVideos bool,
-	duplicateStrategy string,
 	fileHashMap *sync.Map,
 	hashCache *sync.Map,
 ) {
@@ -36,6 +33,9 @@ func creator(
 	var wg sync.WaitGroup
 
 	numWorkers := runtime.NumCPU() / 2
+	if numWorkers < 1 {
+		numWorkers = 1
+	}
 
 	for i := 0; i < numWorkers; i++ {
 		wg.Add(1)
@@ -52,7 +52,6 @@ func creator(
 					fileTypesToInclude,
 					organisePhotos,
 					organiseVideos,
-					duplicateStrategy,
 					fileHashMap,
 					hashCache,
 				)
@@ -93,7 +92,6 @@ func processFile(
 	fileTypesToInclude []string,
 	organisePhotos bool,
 	organiseVideos bool,
-	duplicateStrategy string,
 	fileHashMap *sync.Map,
 	hashCache *sync.Map,
 ) {
@@ -110,28 +108,6 @@ func processFile(
 		return
 	}
 
-	isDuplicate, err := duplicate.IsDuplicate(path, duplicateStrategy, fileHashMap, hashCache)
-	if err != nil {
-		errorQueue <- err
-		return
-	}
-
-	if isDuplicate {
-		switch duplicateStrategy {
-		case "skip":
-			fmt.Printf("Skipped duplicate file: %v\n", path)
-			logMoveAction(path, "", true, duplicateStrategy)
-			return
-		case "delete":
-			if err := os.Remove(path); err != nil {
-				errorQueue <- fmt.Errorf("failed to delete duplicate file: %v", err)
-			} else {
-				logMoveAction(path, "", true, duplicateStrategy)
-			}
-			return
-		}
-	}
-
 	if geoLocation {
 		country, err := getCountry(path)
 		if err != nil {
@@ -141,7 +117,7 @@ func processFile(
 			warnQueue <- fmt.Sprintf("no country found for file: %v", path)
 		}
 
-		fileQueue <- FileInfo{Path: path, FileType: fileType, isDuplicate: isDuplicate, Country: country}
+		fileQueue <- FileInfo{Path: path, FileType: fileType, Country: country}
 	} else {
 		createdDate, hasCreationDate, err := getCreatedTime(path)
 		if err != nil {
@@ -152,31 +128,13 @@ func processFile(
 		fileQueue <- FileInfo{
 			Path:            path,
 			FileType:        fileType,
-			isDuplicate:     isDuplicate,
 			Created:         createdDate,
 			HasCreationDate: hasCreationDate,
 		}
 	}
 }
 
 func getFileType(path string, fileTypesToInclude []string, organisePhotos bool, organiseVideos bool) FileType {
-	file, err := os.Open(path)
-	if err != nil {
-		logger(LoggerTypeWarning, fmt.Sprintf("failed to open file %v: %v", path, err))
-		return FileTypeUnknown
-	}
-	defer file.Close()
-
-	fileInfo, err := file.Stat()
-	if err != nil {
-		logger(LoggerTypeWarning, fmt.Sprintf("failed to get file info: %v", err))
-		return FileTypeUnknown
-	}
-
-	if fileInfo.IsDir() {
-		return FileTypeFolder
-	}
-
 	fileType := FileTypeUnknown
 	if fileTypesToInclude != nil {
 		fileType = FileTypeExcluded

app/creator_test.go

@@ -0,0 +1,22 @@
+package main
+
+import "testing"
+
+func TestGetFileType_ExtensionOnly(t *testing.T) {
+	if got := getFileType("C:/tmp/a.JPG", nil, true, true); got != FileTypeImage {
+		t.Fatalf("expected image, got %v", got)
+	}
+	if got := getFileType("C:/tmp/a.MP4", nil, true, true); got != FileTypeVideo {
+		t.Fatalf("expected video, got %v", got)
+	}
+
+	// When types list is provided and does not include extension, excluded.
+	if got := getFileType("C:/tmp/a.jpg", []string{".mp4"}, true, true); got != FileTypeExcluded {
+		t.Fatalf("expected excluded, got %v", got)
+	}
+
+	// Unknown extension becomes Unknown when no explicit types filter.
+	if got := getFileType("C:/tmp/a.bin", nil, true, true); got != FileTypeUnknown {
+		t.Fatalf("expected unknown, got %v", got)
+	}
+}

app/file.go

@@ -33,6 +33,10 @@ func getPhotoType(fileExt string) PhotoType {
 		return PNG
 	case ".gif":
 		return GIF
+	case ".dng":
+		return DNG
+	case ".nef":
+		return NEF
 	default:
 		return -1
 	}

app/file_test.go

@@ -0,0 +1,22 @@
+package main
+
+import "testing"
+
+func TestPhotoAndVideoDetection(t *testing.T) {
+	if !isPhoto(".JPG") {
+		t.Fatalf("expected .JPG to be photo")
+	}
+	if !isPhoto(".jpeg") {
+		t.Fatalf("expected .jpeg to be photo")
+	}
+	if isPhoto(".txt") {
+		t.Fatalf("expected .txt to not be photo")
+	}
+
+	if !isVideo(".mp4") {
+		t.Fatalf("expected .mp4 to be video")
+	}
+	if isVideo(".jpg") {
+		t.Fatalf("expected .jpg to not be video")
+	}
+}