Compare commits

..

No commits in common. "master" and "v5.1.1" have entirely different histories.

7 changed files with 185 additions and 284 deletions

View File

@ -8,10 +8,10 @@ chkbit is a tool that ensures the safety of your files by checking if their *dat
- [How it works](#how-it-works)
- [Installation](#installation)
- [chkbit as a Go module](#chkbit-as-a-go-module)
- [Usage](#usage)
- [Repair](#repair)
- [Ignore files](#ignore-files)
- [chkbit as a Go module](#chkbit-as-a-go-module)
- [FAQ](#faq)
@ -50,7 +50,7 @@ Building from the source requires Go.
- Either install it directly
```shell
go install github.com/laktak/chkbit/v5/cmd/chkbit@latest
go install github.com/laktak/chkbit@latest
```
- or clone and build
@ -58,11 +58,22 @@ go install github.com/laktak/chkbit/v5/cmd/chkbit@latest
```shell
git clone https://github.com/laktak/chkbit
chkbit/scripts/build
# binary:
ls -l chkbit/chkbit
# output is here:
chkbit/chkbit
```
## chkbit as a Go module
chkbit is can also be used in other Go programs.
```
go get github.com/laktak/chkbit
```
For more information see the linked documentation on [pkg.go.dev](https://pkg.go.dev/github.com/laktak/chkbit).
## Usage
Run `chkbit -u PATH` to create/update the chkbit index.
@ -84,18 +95,15 @@ Arguments:
Flags:
-h, --help Show context-sensitive help.
-H, --tips Show tips.
-c, --check check mode: chkbit will verify files in readonly mode (default mode)
-u, --update update mode: add and update indices
-a, --add-only add mode: only add new files, do not check existing (quicker)
-i, --show-ignored-only show-ignored mode: only show ignored files
-u, --update update indices (without this chkbit will verify files in readonly mode)
--show-ignored-only only show ignored files
-m, --show-missing show missing files/directories
--force force update of damaged items (advanced usage only)
-S, --skip-symlinks do not follow symlinks
-R, --no-recurse do not recurse into subdirectories
--algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3)
-f, --force force update of damaged items
-s, --skip-symlinks do not follow symlinks
-D, --no-dir-in-index do not track directories in the index
-l, --log-file=STRING write to a logfile if specified
--log-verbose verbose logging
--algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3)
--index-name=".chkbit" filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
--ignore-name=".chkbitignore" filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)
-w, --workers=5 number of workers to use (default: 5)
@ -155,17 +163,6 @@ Add a `.chkbitignore` file containing the names of the files/directories you wis
- hidden files (starting with a `.`) are ignored by default
## chkbit as a Go module
chkbit is can also be used in other Go programs.
```
go get github.com/laktak/chkbit/v5
```
For more information see the documentation on [pkg.go.dev](https://pkg.go.dev/github.com/laktak/chkbit/v5).
## FAQ
### Should I run `chkbit` on my whole drive?

View File

@ -1,6 +1,6 @@
package main
var headerHelp = `Checks the data integrity of your files.
var headerHelp = `Checks the data integrity of your files.
For help tips run "chkbit -H" or go to
https://github.com/laktak/chkbit
`

View File

@ -10,8 +10,8 @@ import (
"time"
"github.com/alecthomas/kong"
"github.com/laktak/chkbit/v5"
"github.com/laktak/chkbit/v5/cmd/chkbit/util"
"github.com/laktak/chkbit"
"github.com/laktak/chkbit/cmd/chkbit/util"
"github.com/laktak/lterm"
)
@ -44,18 +44,15 @@ var (
var cli struct {
Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"`
Tips bool `short:"H" help:"Show tips."`
Check bool `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)"`
Update bool `short:"u" help:"update mode: add and update indices"`
AddOnly bool `short:"a" help:"add mode: only add new files, do not check existing (quicker)"`
ShowIgnoredOnly bool `short:"i" help:"show-ignored mode: only show ignored files"`
Update bool `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"`
ShowIgnoredOnly bool `help:"only show ignored files"`
ShowMissing bool `short:"m" help:"show missing files/directories"`
Force bool `help:"force update of damaged items (advanced usage only)"`
SkipSymlinks bool `short:"S" help:"do not follow symlinks"`
NoRecurse bool `short:"R" help:"do not recurse into subdirectories"`
Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
Force bool `short:"f" help:"force update of damaged items"`
SkipSymlinks bool `short:"s" help:"do not follow symlinks"`
NoDirInIndex bool `short:"D" help:"do not track directories in the index"`
LogFile string `short:"l" help:"write to a logfile if specified"`
LogVerbose bool `help:"verbose logging"`
Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
IgnoreName string `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"`
Workers int `short:"w" default:"5" help:"number of workers to use (default: 5)"`
@ -157,11 +154,9 @@ func (m *Main) showStatus() {
}
func (m *Main) process() bool {
// verify mode
var b01 = map[bool]int8{false: 0, true: 1}
if b01[cli.Check]+b01[cli.Update]+b01[cli.AddOnly]+b01[cli.ShowIgnoredOnly] > 1 {
fmt.Println("Error: can only run one mode at a time!")
os.Exit(1)
if cli.Update && cli.ShowIgnoredOnly {
fmt.Println("Error: use either --update or --show-ignored-only!")
return false
}
var err error
@ -171,12 +166,10 @@ func (m *Main) process() bool {
return false
}
m.context.ForceUpdateDmg = cli.Force
m.context.UpdateIndex = cli.Update || cli.AddOnly
m.context.AddOnly = cli.AddOnly
m.context.UpdateIndex = cli.Update
m.context.ShowIgnoredOnly = cli.ShowIgnoredOnly
m.context.ShowMissing = cli.ShowMissing
m.context.SkipSymlinks = cli.SkipSymlinks
m.context.SkipSubdirectories = cli.NoRecurse
m.context.TrackDirectories = !cli.NoDirInIndex
var wg sync.WaitGroup

View File

@ -8,22 +8,20 @@ import (
)
type Context struct {
NumWorkers int
UpdateIndex bool
AddOnly bool
ShowIgnoredOnly bool
ShowMissing bool
ForceUpdateDmg bool
HashAlgo string
TrackDirectories bool
SkipSymlinks bool
SkipSubdirectories bool
IndexFilename string
IgnoreFilename string
WorkQueue chan *WorkItem
LogQueue chan *LogEvent
PerfQueue chan *PerfEvent
wg sync.WaitGroup
NumWorkers int
UpdateIndex bool
ShowIgnoredOnly bool
ShowMissing bool
ForceUpdateDmg bool
HashAlgo string
TrackDirectories bool
SkipSymlinks bool
IndexFilename string
IgnoreFilename string
WorkQueue chan *WorkItem
LogQueue chan *LogEvent
PerfQueue chan *PerfEvent
wg sync.WaitGroup
mutex sync.Mutex
NumTotal int
@ -72,9 +70,7 @@ func (context *Context) log(stat Status, message string) {
context.NumTotal++
context.NumNew++
case STATUS_OK:
if !context.AddOnly {
context.NumTotal++
}
context.NumTotal++
case STATUS_MISSING:
context.NumDel++
//case STATUS_PANIC:
@ -187,9 +183,7 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) {
context.addWork(root, filesToIndex, dirList, ignore)
if !context.SkipSubdirectories {
for _, name := range dirList {
context.scanDir(filepath.Join(root, name), ignore)
}
for _, name := range dirList {
context.scanDir(filepath.Join(root, name), ignore)
}
}

2
go.mod
View File

@ -1,4 +1,4 @@
module github.com/laktak/chkbit/v5
module github.com/laktak/chkbit
go 1.22.3

View File

@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"slices"
)
@ -68,6 +69,10 @@ func (i *Index) getIndexFilepath() string {
return filepath.Join(i.path, i.context.IndexFilename)
}
func (i *Index) setMod(value bool) {
i.modified = value
}
func (i *Index) logFilePanic(name string, message string) {
i.context.log(STATUS_PANIC, filepath.Join(i.path, name)+": "+message)
}
@ -91,7 +96,7 @@ func (i *Index) calcHashes(ignore *Ignore) {
var info *idxInfo
algo := i.context.HashAlgo
if val, ok := i.cur[name]; ok {
// existing file
// existing
if val.LegacyHash != nil {
// convert from py1 to new format
val = idxInfo{
@ -104,13 +109,8 @@ func (i *Index) calcHashes(ignore *Ignore) {
if val.Algo != nil {
algo = *val.Algo
}
if i.context.AddOnly {
info = &val
} else {
info, err = i.calcFile(name, algo)
}
info, err = i.calcFile(name, algo)
} else {
// new file
if i.readonly {
info = &idxInfo{Algo: &algo}
} else {
@ -137,14 +137,14 @@ func (i *Index) checkFix(forceUpdateDmg bool) {
for name, b := range i.new {
if a, ok := i.cur[name]; !ok {
i.logFile(STATUS_NEW, name)
i.modified = true
i.setMod(true)
} else {
amod := int64(a.ModTime)
bmod := int64(b.ModTime)
if a.Hash != nil && b.Hash != nil && *a.Hash == *b.Hash {
i.logFile(STATUS_OK, name)
if amod != bmod {
i.modified = true
i.setMod(true)
}
continue
}
@ -155,45 +155,37 @@ func (i *Index) checkFix(forceUpdateDmg bool) {
// keep DMG entry
i.new[name] = a
} else {
i.modified = true
i.setMod(true)
}
} else if amod < bmod {
i.logFile(STATUS_UPDATE, name)
i.modified = true
i.setMod(true)
} else if amod > bmod {
i.logFile(STATUS_UP_WARN_OLD, name)
i.modified = true
i.setMod(true)
}
}
}
// track missing
for name := range i.cur {
if _, ok := i.new[name]; !ok {
i.modified = true
if i.context.ShowMissing {
if i.context.ShowMissing {
for name := range i.cur {
if _, ok := i.new[name]; !ok {
i.logFile(STATUS_MISSING, name)
i.setMod(true)
}
}
}
// dirs
m := make(map[string]bool)
for _, n := range i.newDirList {
m[n] = true
}
for _, name := range i.curDirList {
if !m[name] {
i.modified = true
if i.context.ShowMissing {
// dirs
m := make(map[string]bool)
for _, n := range i.newDirList {
m[n] = true
}
for _, name := range i.curDirList {
if !m[name] {
i.logDir(STATUS_MISSING, name+"/")
i.setMod(true)
}
}
}
if len(i.newDirList) != len(i.curDirList) {
// added
i.modified = true
}
}
}
func (i *Index) calcFile(name string, a string) (*idxInfo, error) {
@ -239,7 +231,7 @@ func (i *Index) save() (bool, error) {
if err != nil {
return false, err
}
i.modified = false
i.setMod(false)
return true, nil
} else {
return false, nil
@ -249,11 +241,13 @@ func (i *Index) save() (bool, error) {
func (i *Index) load() error {
if _, err := os.Stat(i.getIndexFilepath()); err != nil {
if os.IsNotExist(err) {
// todo
i.setMod(true)
return nil
}
return err
}
i.modified = false
i.setMod(false)
file, err := os.ReadFile(i.getIndexFilepath())
if err != nil {
return err
@ -275,7 +269,7 @@ func (i *Index) load() error {
} else {
}
if data.IdxHash != hashMd5(text) {
i.modified = true
i.setMod(true)
i.logFile(STATUS_ERR_IDX, i.getIndexFilepath())
}
} else {
@ -292,12 +286,12 @@ func (i *Index) load() error {
}
}
}
// dirs
if data.Dir != nil {
slices.Sort(data.Dir)
i.curDirList = data.Dir
if i.context.TrackDirectories && !reflect.DeepEqual(i.curDirList, i.newDirList) {
i.setMod(true)
}
}
return nil
}

View File

@ -67,12 +67,7 @@ func setDate(filename string, r int) {
os.Chtimes(filename, date, date)
}
func genFile(path string, size int) {
os.WriteFile(path, make([]byte, size), 0644)
setDate(path, size*size)
}
func genFiles(dir string, a int) {
func genFile(dir string, a int) {
os.MkdirAll(dir, 0755)
for i := 1; i <= 5; i++ {
size := a*i*wordIdx*100 + extIdx
@ -83,7 +78,9 @@ func genFiles(dir string, a int) {
}
file += "." + nextExt()
genFile(filepath.Join(dir, file), size)
path := filepath.Join(dir, file)
os.WriteFile(path, make([]byte, size), 0644)
setDate(path, size*size)
}
}
@ -92,11 +89,11 @@ func genDir(root string) {
for i := 1; i <= 5; i++ {
dir := filepath.Join(root, start, nextWord())
genFiles(dir, 1)
genFile(dir, 1)
if wordIdx%3 == 0 {
dir = filepath.Join(dir, nextWord())
genFiles(dir, 1)
genFile(dir, 1)
}
}
}
@ -118,8 +115,6 @@ func setupMiscFiles() {
genDir(root)
os.MkdirAll(filepath.Join(root, "day/car/empty"), 0755)
rootPeople := filepath.Join(root, "people")
testPeople := filepath.Join(testDir, "people")
@ -142,124 +137,60 @@ func TestRoot(t *testing.T) {
tool := getCmd()
root := filepath.Join(testDir, "root")
// update index, no recourse
t.Run("no-recourse", func(t *testing.T) {
cmd := exec.Command(tool, "-umR", filepath.Join(root, "day/office"))
out, err := cmd.Output()
// step1: update index
cmd := exec.Command(tool, "-um", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("step1 failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "67 directories were updated")
checkOut(t, sout, "300 file hashes were added")
checkNotOut(t, sout, "removed")
// step2: delete files, check for missing
os.RemoveAll(filepath.Join(root, "thing/change"))
os.Remove(filepath.Join(root, "time/hour/minute/body-information.csv"))
cmd = exec.Command(tool, "-m", root)
out, err = cmd.Output()
if err != nil {
t.Fatalf("step2 failed with '%s'\n", err)
}
sout = string(out)
checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
checkOut(t, sout, "2 files/directories would have been removed")
// step2a: do not report missing without -m
cmd = exec.Command(tool, root)
out, err = cmd.Output()
if err != nil {
t.Fatalf("step2a failed with '%s'\n", err)
}
sout = string(out)
checkNotOut(t, sout, "del ")
checkNotOut(t, sout, "removed")
// step3: check for missing and update
cmd = exec.Command(tool, "-um", root)
out, err = cmd.Output()
if err != nil {
t.Fatalf("step3 failed with '%s'\n", err)
}
sout = string(out)
checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
checkOut(t, sout, "2 files/directories have been removed")
// step4: check again
for i := 0; i < 10; i++ {
cmd = exec.Command(tool, "-u", root)
out, err = cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
t.Fatalf("step4 failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "Processed 5 files")
checkOut(t, sout, "- 1 directory was updated")
checkOut(t, sout, "- 5 file hashes were added")
checkOut(t, sout, "- 0 file hashes were updated")
checkNotOut(t, sout, "removed")
})
// update remaining index from root
t.Run("update-remaining", func(t *testing.T) {
cmd := exec.Command(tool, "-um", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "Processed 300 files")
checkOut(t, sout, "- 66 directories were updated")
checkOut(t, sout, "- 295 file hashes were added")
checkOut(t, sout, "- 0 file hashes were updated")
checkNotOut(t, sout, "removed")
})
// delete files, check for missing
t.Run("delete", func(t *testing.T) {
os.RemoveAll(filepath.Join(root, "thing/change"))
os.Remove(filepath.Join(root, "time/hour/minute/body-information.csv"))
cmd := exec.Command(tool, "-m", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
checkOut(t, sout, "2 files/directories would have been removed")
})
// do not report missing without -m
t.Run("no-missing", func(t *testing.T) {
cmd := exec.Command(tool, root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkNotOut(t, sout, "del ")
checkNotOut(t, sout, "removed")
})
// check for missing and update
t.Run("missing", func(t *testing.T) {
cmd := exec.Command(tool, "-um", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "del /tmp/chkbit/root/thing/change/")
checkOut(t, sout, "2 files/directories have been removed")
})
// check again
t.Run("repeat", func(t *testing.T) {
for i := 0; i < 10; i++ {
cmd := exec.Command(tool, "-uv", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "Processed 289 files")
checkNotOut(t, sout, "removed")
checkNotOut(t, sout, "updated")
checkNotOut(t, sout, "added")
}
})
// add files only
t.Run("add-only", func(t *testing.T) {
genFiles(filepath.Join(root, "way/add"), 99)
genFile(filepath.Join(root, "time/add-file.txt"), 500)
// modify existing, will not be reported:
genFile(filepath.Join(root, "way/job/word-business.mp3"), 500)
cmd := exec.Command(tool, "-a", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "Processed 6 files")
checkOut(t, sout, "- 3 directories were updated")
checkOut(t, sout, "- 6 file hashes were added")
checkOut(t, sout, "- 0 file hashes were updated")
})
// update remaining
t.Run("update-remaining-add", func(t *testing.T) {
cmd := exec.Command(tool, "-u", root)
out, err := cmd.Output()
if err != nil {
t.Fatalf("failed with '%s'\n", err)
}
sout := string(out)
checkOut(t, sout, "Processed 295 files")
checkOut(t, sout, "- 1 directory was updated")
checkOut(t, sout, "- 0 file hashes were added")
checkOut(t, sout, "- 1 file hash was updated")
})
sout = string(out)
checkOut(t, sout, "Processed 289 files")
}
}
func TestDMG(t *testing.T) {
@ -285,58 +216,50 @@ func TestDMG(t *testing.T) {
t2, _ := time.Parse(time.RFC3339, "2022-02-01T12:00:00Z")
t3, _ := time.Parse(time.RFC3339, "2022-02-01T13:00:00Z")
// create test and set the modified time"
t.Run("create", func(t *testing.T) {
os.WriteFile(testFile, []byte("foo1"), 0644)
os.Chtimes(testFile, t2, t2)
// step1: create test and set the modified time"
os.WriteFile(testFile, []byte("foo1"), 0644)
os.Chtimes(testFile, t2, t2)
cmd := exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("failed with '%s'\n", err)
} else {
checkOut(t, string(out), "new test.txt")
cmd := exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("step1 failed with '%s'\n", err)
} else {
checkOut(t, string(out), "new test.txt")
}
// step2: update test with different content & old modified (expect 'old')"
os.WriteFile(testFile, []byte("foo2"), 0644)
os.Chtimes(testFile, t1, t1)
cmd = exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("step2 failed with '%s'\n", err)
} else {
checkOut(t, string(out), "old test.txt")
}
// step3: update test & new modified (expect 'upd')"
os.WriteFile(testFile, []byte("foo3"), 0644)
os.Chtimes(testFile, t3, t3)
cmd = exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("step3 failed with '%s'\n", err)
} else {
checkOut(t, string(out), "upd test.txt")
}
// step4: Now update test with the same modified to simulate damage (expect DMG)"
os.WriteFile(testFile, []byte("foo4"), 0644)
os.Chtimes(testFile, t3, t3)
cmd = exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
if cmd.ProcessState.ExitCode() != 1 {
t.Fatalf("step4 expected to fail with exit code 1 vs %d!", cmd.ProcessState.ExitCode())
}
})
// update test with different content & old modified (expect 'old')"
t.Run("expect-old", func(t *testing.T) {
os.WriteFile(testFile, []byte("foo2"), 0644)
os.Chtimes(testFile, t1, t1)
cmd := exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("failed with '%s'\n", err)
} else {
checkOut(t, string(out), "old test.txt")
}
})
// update test & new modified (expect 'upd')"
t.Run("expect-upd", func(t *testing.T) {
os.WriteFile(testFile, []byte("foo3"), 0644)
os.Chtimes(testFile, t3, t3)
cmd := exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
t.Fatalf("failed with '%s'\n", err)
} else {
checkOut(t, string(out), "upd test.txt")
}
})
// Now update test with the same modified to simulate damage (expect DMG)"
t.Run("expect-DMG", func(t *testing.T) {
os.WriteFile(testFile, []byte("foo4"), 0644)
os.Chtimes(testFile, t3, t3)
cmd := exec.Command(tool, "-u", ".")
if out, err := cmd.Output(); err != nil {
if cmd.ProcessState.ExitCode() != 1 {
t.Fatalf("expected to fail with exit code 1 vs %d!", cmd.ProcessState.ExitCode())
}
checkOut(t, string(out), "DMG test.txt")
} else {
t.Fatal("expected to fail!")
}
})
checkOut(t, string(out), "DMG test.txt")
} else {
t.Fatal("step4 expected to fail!")
}
}