new --add-only mode

README.md

@@ -8,10 +8,10 @@ chkbit is a tool that ensures the safety of your files by checking if their *dat
 
 - [How it works](#how-it-works)
 - [Installation](#installation)
-- [chkbit as a Go module](#chkbit-as-a-go-module)
 - [Usage](#usage)
 - [Repair](#repair)
 - [Ignore files](#ignore-files)
+- [chkbit as a Go module](#chkbit-as-a-go-module)
 - [FAQ](#faq)
 
 
@@ -50,7 +50,7 @@ Building from the source requires Go.
 - Either install it directly
 
 ```shell
-go install github.com/laktak/chkbit@latest
+go install github.com/laktak/chkbit/v5/cmd/chkbit@latest
 ```
 
 - or clone and build
@@ -58,22 +58,11 @@ go install github.com/laktak/chkbit@latest
 ```shell
 git clone https://github.com/laktak/chkbit
 chkbit/scripts/build
-# output is here:
+# binary:
-chkbit/chkbit
+ls -l chkbit/chkbit
 ```
 
 
-## chkbit as a Go module
-
-chkbit is can also be used in other Go programs.
-
-```
-go get github.com/laktak/chkbit
-```
-
-For more information see the linked documentation on [pkg.go.dev](https://pkg.go.dev/github.com/laktak/chkbit).
-
-
 ## Usage
 
 Run `chkbit -u PATH` to create/update the chkbit index.
@@ -95,15 +84,18 @@ Arguments:
 Flags:
   -h, --help                           Show context-sensitive help.
   -H, --tips                           Show tips.
-  -u, --update                         update indices (without this chkbit will verify files in readonly mode)
+  -c, --check                          check mode: chkbit will verify files in readonly mode (default mode)
-      --show-ignored-only              only show ignored files
+  -u, --update                         update mode: add and update indices
+  -a, --add-only                       add mode: only add new files, do not check existing (quicker)
+  -i, --show-ignored-only              show-ignored mode: only show ignored files
   -m, --show-missing                   show missing files/directories
-      --algo="blake3"                  hash algorithm: md5, sha512, blake3 (default: blake3)
+      --force                          force update of damaged items (advanced usage only)
-  -f, --force                          force update of damaged items
+  -S, --skip-symlinks                  do not follow symlinks
-  -s, --skip-symlinks                  do not follow symlinks
+  -R, --no-recurse                     do not recurse into subdirectories
   -D, --no-dir-in-index                do not track directories in the index
   -l, --log-file=STRING                write to a logfile if specified
       --log-verbose                    verbose logging
+      --algo="blake3"                  hash algorithm: md5, sha512, blake3 (default: blake3)
       --index-name=".chkbit"           filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
       --ignore-name=".chkbitignore"    filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)
   -w, --workers=5                      number of workers to use (default: 5)
@@ -163,6 +155,17 @@ Add a `.chkbitignore` file containing the names of the files/directories you wis
 - hidden files (starting with a `.`) are ignored by default
 
 
+## chkbit as a Go module
+
+chkbit is can also be used in other Go programs.
+
+```
+go get github.com/laktak/chkbit/v5
+```
+
+For more information see the documentation on [pkg.go.dev](https://pkg.go.dev/github.com/laktak/chkbit/v5).
+
+
 ## FAQ
 
 ### Should I run `chkbit` on my whole drive?

cmd/chkbit/main.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/alecthomas/kong"
-	"github.com/laktak/chkbit"
+	"github.com/laktak/chkbit/v5"
-	"github.com/laktak/chkbit/cmd/chkbit/util"
+	"github.com/laktak/chkbit/v5/cmd/chkbit/util"
 	"github.com/laktak/lterm"
 )
 
@@ -44,15 +44,18 @@ var (
 var cli struct {
 	Paths           []string `arg:"" optional:"" name:"paths" help:"directories to check"`
 	Tips            bool     `short:"H" help:"Show tips."`
-	Update          bool     `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"`
+	Check           bool     `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)"`
-	ShowIgnoredOnly bool     `help:"only show ignored files"`
+	Update          bool     `short:"u" help:"update mode: add and update indices"`
+	AddOnly         bool     `short:"a" help:"add mode: only add new files, do not check existing (quicker)"`
+	ShowIgnoredOnly bool     `short:"i" help:"show-ignored mode: only show ignored files"`
 	ShowMissing     bool     `short:"m" help:"show missing files/directories"`
-	Algo            string   `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
+	Force           bool     `help:"force update of damaged items (advanced usage only)"`
-	Force           bool     `short:"f" help:"force update of damaged items"`
+	SkipSymlinks    bool     `short:"S" help:"do not follow symlinks"`
-	SkipSymlinks    bool     `short:"s" help:"do not follow symlinks"`
+	NoRecurse       bool     `short:"R" help:"do not recurse into subdirectories"`
 	NoDirInIndex    bool     `short:"D" help:"do not track directories in the index"`
 	LogFile         string   `short:"l" help:"write to a logfile if specified"`
 	LogVerbose      bool     `help:"verbose logging"`
+	Algo            string   `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
 	IndexName       string   `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
 	IgnoreName      string   `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"`
 	Workers         int      `short:"w" default:"5" help:"number of workers to use (default: 5)"`
@@ -154,9 +157,11 @@ func (m *Main) showStatus() {
 }
 
 func (m *Main) process() bool {
-	if cli.Update && cli.ShowIgnoredOnly {
+	// verify mode
-		fmt.Println("Error: use either --update or --show-ignored-only!")
+	var b01 = map[bool]int8{false: 0, true: 1}
-		return false
+	if b01[cli.Check]+b01[cli.Update]+b01[cli.AddOnly]+b01[cli.ShowIgnoredOnly] > 1 {
+		fmt.Println("Error: can only run one mode at a time!")
+		os.Exit(1)
 	}
 
 	var err error
@@ -166,10 +171,12 @@ func (m *Main) process() bool {
 		return false
 	}
 	m.context.ForceUpdateDmg = cli.Force
-	m.context.UpdateIndex = cli.Update
+	m.context.UpdateIndex = cli.Update || cli.AddOnly
+	m.context.AddOnly = cli.AddOnly
 	m.context.ShowIgnoredOnly = cli.ShowIgnoredOnly
 	m.context.ShowMissing = cli.ShowMissing
 	m.context.SkipSymlinks = cli.SkipSymlinks
+	m.context.SkipSubdirectories = cli.NoRecurse
 	m.context.TrackDirectories = !cli.NoDirInIndex
 
 	var wg sync.WaitGroup

context.go

@@ -10,12 +10,14 @@ import (
 type Context struct {
 	NumWorkers         int
 	UpdateIndex        bool
+	AddOnly            bool
 	ShowIgnoredOnly    bool
 	ShowMissing        bool
 	ForceUpdateDmg     bool
 	HashAlgo           string
 	TrackDirectories   bool
 	SkipSymlinks       bool
+	SkipSubdirectories bool
 	IndexFilename      string
 	IgnoreFilename     string
 	WorkQueue          chan *WorkItem
@@ -70,7 +72,9 @@ func (context *Context) log(stat Status, message string) {
 		context.NumTotal++
 		context.NumNew++
 	case STATUS_OK:
+		if !context.AddOnly {
 			context.NumTotal++
+		}
 	case STATUS_MISSING:
 		context.NumDel++
 		//case STATUS_PANIC:
@@ -183,7 +187,9 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) {
 
 	context.addWork(root, filesToIndex, dirList, ignore)
 
+	if !context.SkipSubdirectories {
 		for _, name := range dirList {
 			context.scanDir(filepath.Join(root, name), ignore)
 		}
 	}
+}

go.mod

@@ -1,4 +1,4 @@
-module github.com/laktak/chkbit
+module github.com/laktak/chkbit/v5
 
 go 1.22.3
 

index.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
-	"reflect"
 	"slices"
 )
 
@@ -69,10 +68,6 @@ func (i *Index) getIndexFilepath() string {
 	return filepath.Join(i.path, i.context.IndexFilename)
 }
 
-func (i *Index) setMod(value bool) {
-	i.modified = value
-}
-
 func (i *Index) logFilePanic(name string, message string) {
 	i.context.log(STATUS_PANIC, filepath.Join(i.path, name)+": "+message)
 }
@@ -96,7 +91,7 @@ func (i *Index) calcHashes(ignore *Ignore) {
 		var info *idxInfo
 		algo := i.context.HashAlgo
 		if val, ok := i.cur[name]; ok {
-			// existing
+			// existing file
 			if val.LegacyHash != nil {
 				// convert from py1 to new format
 				val = idxInfo{
@@ -109,8 +104,13 @@ func (i *Index) calcHashes(ignore *Ignore) {
 			if val.Algo != nil {
 				algo = *val.Algo
 			}
-			info, err = i.calcFile(name, algo)
+			if i.context.AddOnly {
+				info = &val
 			} else {
+				info, err = i.calcFile(name, algo)
+			}
+		} else {
+			// new file
 			if i.readonly {
 				info = &idxInfo{Algo: &algo}
 			} else {
@@ -137,14 +137,14 @@ func (i *Index) checkFix(forceUpdateDmg bool) {
 	for name, b := range i.new {
 		if a, ok := i.cur[name]; !ok {
 			i.logFile(STATUS_NEW, name)
-			i.setMod(true)
+			i.modified = true
 		} else {
 			amod := int64(a.ModTime)
 			bmod := int64(b.ModTime)
 			if a.Hash != nil && b.Hash != nil && *a.Hash == *b.Hash {
 				i.logFile(STATUS_OK, name)
 				if amod != bmod {
-					i.setMod(true)
+					i.modified = true
 				}
 				continue
 			}
@@ -155,24 +155,27 @@ func (i *Index) checkFix(forceUpdateDmg bool) {
 					// keep DMG entry
 					i.new[name] = a
 				} else {
-					i.setMod(true)
+					i.modified = true
 				}
 			} else if amod < bmod {
 				i.logFile(STATUS_UPDATE, name)
-				i.setMod(true)
+				i.modified = true
 			} else if amod > bmod {
 				i.logFile(STATUS_UP_WARN_OLD, name)
-				i.setMod(true)
+				i.modified = true
 			}
 		}
 	}
-	if i.context.ShowMissing {
+	// track missing
 	for name := range i.cur {
 		if _, ok := i.new[name]; !ok {
+			i.modified = true
+			if i.context.ShowMissing {
 				i.logFile(STATUS_MISSING, name)
-				i.setMod(true)
 			}
 		}
+	}
+
 	// dirs
 	m := make(map[string]bool)
 	for _, n := range i.newDirList {
@@ -180,13 +183,18 @@ func (i *Index) checkFix(forceUpdateDmg bool) {
 	}
 	for _, name := range i.curDirList {
 		if !m[name] {
+			i.modified = true
+			if i.context.ShowMissing {
 				i.logDir(STATUS_MISSING, name+"/")
-				i.setMod(true)
 			}
 		}
+	}
+	if len(i.newDirList) != len(i.curDirList) {
+		// added
+		i.modified = true
+	}
 
 }
-}
 
 func (i *Index) calcFile(name string, a string) (*idxInfo, error) {
 	path := filepath.Join(i.path, name)
@@ -231,7 +239,7 @@ func (i *Index) save() (bool, error) {
 		if err != nil {
 			return false, err
 		}
-		i.setMod(false)
+		i.modified = false
 		return true, nil
 	} else {
 		return false, nil
@@ -241,13 +249,11 @@ func (i *Index) save() (bool, error) {
 func (i *Index) load() error {
 	if _, err := os.Stat(i.getIndexFilepath()); err != nil {
 		if os.IsNotExist(err) {
-			// todo
-			i.setMod(true)
 			return nil
 		}
 		return err
 	}
-	i.setMod(false)
+	i.modified = false
 	file, err := os.ReadFile(i.getIndexFilepath())
 	if err != nil {
 		return err
@@ -269,7 +275,7 @@ func (i *Index) load() error {
 		} else {
 		}
 		if data.IdxHash != hashMd5(text) {
-			i.setMod(true)
+			i.modified = true
 			i.logFile(STATUS_ERR_IDX, i.getIndexFilepath())
 		}
 	} else {
@@ -286,12 +292,12 @@ func (i *Index) load() error {
 			}
 		}
 	}
+
+	// dirs
 	if data.Dir != nil {
 		slices.Sort(data.Dir)
 		i.curDirList = data.Dir
-		if i.context.TrackDirectories && !reflect.DeepEqual(i.curDirList, i.newDirList) {
-			i.setMod(true)
-		}
 	}
+
 	return nil
 }

scripts/run_test.go

@@ -67,7 +67,12 @@ func setDate(filename string, r int) {
 	os.Chtimes(filename, date, date)
 }
 
-func genFile(dir string, a int) {
+func genFile(path string, size int) {
+	os.WriteFile(path, make([]byte, size), 0644)
+	setDate(path, size*size)
+}
+
+func genFiles(dir string, a int) {
 	os.MkdirAll(dir, 0755)
 	for i := 1; i <= 5; i++ {
 		size := a*i*wordIdx*100 + extIdx
@@ -78,9 +83,7 @@ func genFile(dir string, a int) {
 		}
 
 		file += "." + nextExt()
-		path := filepath.Join(dir, file)
+		genFile(filepath.Join(dir, file), size)
-		os.WriteFile(path, make([]byte, size), 0644)
-		setDate(path, size*size)
 	}
 }
 
@@ -89,11 +92,11 @@ func genDir(root string) {
 
 		for i := 1; i <= 5; i++ {
 			dir := filepath.Join(root, start, nextWord())
-			genFile(dir, 1)
+			genFiles(dir, 1)
 
 			if wordIdx%3 == 0 {
 				dir = filepath.Join(dir, nextWord())
-				genFile(dir, 1)
+				genFiles(dir, 1)
 			}
 		}
 	}
@@ -115,6 +118,8 @@ func setupMiscFiles() {
 
 	genDir(root)
 
+	os.MkdirAll(filepath.Join(root, "day/car/empty"), 0755)
+
 	rootPeople := filepath.Join(root, "people")
 	testPeople := filepath.Join(testDir, "people")
 
@@ -137,60 +142,124 @@ func TestRoot(t *testing.T) {
 	tool := getCmd()
 	root := filepath.Join(testDir, "root")
 
-	// step1: update index
+	// update index, no recourse
+	t.Run("no-recourse", func(t *testing.T) {
+		cmd := exec.Command(tool, "-umR", filepath.Join(root, "day/office"))
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatalf("failed with '%s'\n", err)
+		}
+		sout := string(out)
+		checkOut(t, sout, "Processed 5 files")
+		checkOut(t, sout, "- 1 directory was updated")
+		checkOut(t, sout, "- 5 file hashes were added")
+		checkOut(t, sout, "- 0 file hashes were updated")
+		checkNotOut(t, sout, "removed")
+	})
+
+	// update remaining index from root
+	t.Run("update-remaining", func(t *testing.T) {
 		cmd := exec.Command(tool, "-um", root)
 		out, err := cmd.Output()
 		if err != nil {
-		t.Fatalf("step1 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		}
 		sout := string(out)
-	checkOut(t, sout, "67 directories were updated")
+		checkOut(t, sout, "Processed 300 files")
-	checkOut(t, sout, "300 file hashes were added")
+		checkOut(t, sout, "- 66 directories were updated")
+		checkOut(t, sout, "- 295 file hashes were added")
+		checkOut(t, sout, "- 0 file hashes were updated")
 		checkNotOut(t, sout, "removed")
+	})
 
-	// step2: delete files, check for missing
+	// delete files, check for missing
+	t.Run("delete", func(t *testing.T) {
 		os.RemoveAll(filepath.Join(root, "thing/change"))
 		os.Remove(filepath.Join(root, "time/hour/minute/body-information.csv"))
 
-	cmd = exec.Command(tool, "-m", root)
+		cmd := exec.Command(tool, "-m", root)
-	out, err = cmd.Output()
+		out, err := cmd.Output()
 		if err != nil {
-		t.Fatalf("step2 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		}
-	sout = string(out)
+		sout := string(out)
 		checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
 		checkOut(t, sout, "2 files/directories would have been removed")
+	})
 
-	// step2a: do not report missing without -m
+	// do not report missing without -m
-	cmd = exec.Command(tool, root)
+	t.Run("no-missing", func(t *testing.T) {
-	out, err = cmd.Output()
+		cmd := exec.Command(tool, root)
+		out, err := cmd.Output()
 		if err != nil {
-		t.Fatalf("step2a failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		}
-	sout = string(out)
+		sout := string(out)
 		checkNotOut(t, sout, "del ")
 		checkNotOut(t, sout, "removed")
+	})
 
-	// step3: check for missing and update
+	// check for missing and update
-	cmd = exec.Command(tool, "-um", root)
+	t.Run("missing", func(t *testing.T) {
-	out, err = cmd.Output()
+		cmd := exec.Command(tool, "-um", root)
+		out, err := cmd.Output()
 		if err != nil {
-		t.Fatalf("step3 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		}
-	sout = string(out)
+		sout := string(out)
 		checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
 		checkOut(t, sout, "2 files/directories have been removed")
+	})
 
-	// step4: check again
+	// check again
+	t.Run("repeat", func(t *testing.T) {
 		for i := 0; i < 10; i++ {
-		cmd = exec.Command(tool, "-u", root)
+			cmd := exec.Command(tool, "-uv", root)
-		out, err = cmd.Output()
+			out, err := cmd.Output()
 			if err != nil {
-			t.Fatalf("step4 failed with '%s'\n", err)
+				t.Fatalf("failed with '%s'\n", err)
 			}
-		sout = string(out)
+			sout := string(out)
 			checkOut(t, sout, "Processed 289 files")
+			checkNotOut(t, sout, "removed")
+			checkNotOut(t, sout, "updated")
+			checkNotOut(t, sout, "added")
 		}
+	})
+
+	// add files only
+	t.Run("add-only", func(t *testing.T) {
+
+		genFiles(filepath.Join(root, "way/add"), 99)
+		genFile(filepath.Join(root, "time/add-file.txt"), 500)
+		// modify existing, will not be reported:
+		genFile(filepath.Join(root, "way/job/word-business.mp3"), 500)
+
+		cmd := exec.Command(tool, "-a", root)
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatalf("failed with '%s'\n", err)
+		}
+		sout := string(out)
+		checkOut(t, sout, "Processed 6 files")
+		checkOut(t, sout, "- 3 directories were updated")
+		checkOut(t, sout, "- 6 file hashes were added")
+		checkOut(t, sout, "- 0 file hashes were updated")
+	})
+
+	// update remaining
+	t.Run("update-remaining-add", func(t *testing.T) {
+		cmd := exec.Command(tool, "-u", root)
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatalf("failed with '%s'\n", err)
+		}
+		sout := string(out)
+		checkOut(t, sout, "Processed 295 files")
+		checkOut(t, sout, "- 1 directory was updated")
+		checkOut(t, sout, "- 0 file hashes were added")
+		checkOut(t, sout, "- 1 file hash was updated")
+	})
 }
 
 func TestDMG(t *testing.T) {
@@ -216,50 +285,58 @@ func TestDMG(t *testing.T) {
 	t2, _ := time.Parse(time.RFC3339, "2022-02-01T12:00:00Z")
 	t3, _ := time.Parse(time.RFC3339, "2022-02-01T13:00:00Z")
 
-	// step1: create test and set the modified time"
+	// create test and set the modified time"
+	t.Run("create", func(t *testing.T) {
 		os.WriteFile(testFile, []byte("foo1"), 0644)
 		os.Chtimes(testFile, t2, t2)
 
 		cmd := exec.Command(tool, "-u", ".")
 		if out, err := cmd.Output(); err != nil {
-		t.Fatalf("step1 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		} else {
 			checkOut(t, string(out), "new test.txt")
 		}
+	})
 
-	// step2: update test with different content & old modified (expect 'old')"
+	// update test with different content & old modified (expect 'old')"
+	t.Run("expect-old", func(t *testing.T) {
 		os.WriteFile(testFile, []byte("foo2"), 0644)
 		os.Chtimes(testFile, t1, t1)
 
-	cmd = exec.Command(tool, "-u", ".")
+		cmd := exec.Command(tool, "-u", ".")
 		if out, err := cmd.Output(); err != nil {
-		t.Fatalf("step2 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		} else {
 			checkOut(t, string(out), "old test.txt")
 		}
+	})
 
-	// step3: update test & new modified (expect 'upd')"
+	// update test & new modified (expect 'upd')"
+	t.Run("expect-upd", func(t *testing.T) {
 		os.WriteFile(testFile, []byte("foo3"), 0644)
 		os.Chtimes(testFile, t3, t3)
 
-	cmd = exec.Command(tool, "-u", ".")
+		cmd := exec.Command(tool, "-u", ".")
 		if out, err := cmd.Output(); err != nil {
-		t.Fatalf("step3 failed with '%s'\n", err)
+			t.Fatalf("failed with '%s'\n", err)
 		} else {
 			checkOut(t, string(out), "upd test.txt")
 		}
+	})
 
-	// step4: Now update test with the same modified to simulate damage (expect DMG)"
+	// Now update test with the same modified to simulate damage (expect DMG)"
+	t.Run("expect-DMG", func(t *testing.T) {
 		os.WriteFile(testFile, []byte("foo4"), 0644)
 		os.Chtimes(testFile, t3, t3)
 
-	cmd = exec.Command(tool, "-u", ".")
+		cmd := exec.Command(tool, "-u", ".")
 		if out, err := cmd.Output(); err != nil {
 			if cmd.ProcessState.ExitCode() != 1 {
-			t.Fatalf("step4 expected to fail with exit code 1 vs %d!", cmd.ProcessState.ExitCode())
+				t.Fatalf("expected to fail with exit code 1 vs %d!", cmd.ProcessState.ExitCode())
 			}
 			checkOut(t, string(out), "DMG test.txt")
 		} else {
-		t.Fatal("step4 expected to fail!")
+			t.Fatal("expected to fail!")
 		}
+	})
 }