robertvokac/chkbit
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Activity

initial go version

Browse Source
This commit is contained in:
Christian Zangl 2024-08-15 23:30:29 +02:00
parent bb33b02b68
commit 9e6a42f626
No known key found for this signature in database
GPG Key ID: 6D468AC36E2A4B3D
29 changed files with 1530 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -0,0 +1,3 @@
# bin
/chkbit
dist

59
README.md
View File

@ -31,7 +31,7 @@ Remember to always maintain multiple backups for comprehensive data protection.
``` ```
brew install chkbit brew install chkbit
``` ```
- Download for [Linux, macOS or Windows](https://github.com/laktak/chkbit-py/releases). - Download for [Linux, macOS or Windows](https://github.com/laktak/chkbit/releases).
## Usage ## Usage
@ -47,45 +47,28 @@ chkbit will
Run `chkbit PATH` to verify only. Run `chkbit PATH` to verify only.
``` ```
usage: chkbit [-h] [-u] [--show-ignored-only] [--algo ALGO] [-f] [-s] [-l FILE] [--log-verbose] [--index-name NAME] [--ignore-name NAME] [-w N] [--plain] [-q] [-v] [PATH ...] Usage: chkbit [<paths> ...] [flags]
Checks the data integrity of your files. See https://github.com/laktak/chkbit-py Arguments:
[<paths> ...] directories to check
positional arguments: Flags:
PATH directories to check -h, --help Show context-sensitive help.
-H, --tips Show tips.
options: -u, --update update indices (without this chkbit will verify files in readonly mode)
-h, --help show this help message and exit --show-ignored-only only show ignored files
-u, --update update indices (without this chkbit will verify files in readonly mode) --algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3)
--show-ignored-only only show ignored files -f, --force force update of damaged items
--algo ALGO hash algorithm: md5, sha512, blake3 (default: blake3) -s, --skip-symlinks do not follow symlinks
-f, --force force update of damaged items -l, --log-file=STRING write to a logfile if specified
-s, --skip-symlinks do not follow symlinks --log-verbose verbose logging
-l FILE, --log-file FILE --index-name=".chkbit" filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
write to a logfile if specified --ignore-name=".chkbitignore" filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)
--log-verbose verbose logging -w, --workers=5 number of workers to use (default: 5)
--index-name NAME filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit) --plain show plain status instead of being fancy
--ignore-name NAME filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore) -q, --quiet quiet, don't show progress/information
-w N, --workers N number of workers to use (default: 5) -v, --verbose verbose output
--plain show plain status instead of being fancy -V, --version show version information
-q, --quiet quiet, don't show progress/information
-v, --verbose verbose output
.chkbitignore rules:
each line should contain exactly one name
you may use Unix shell-style wildcards (see README)
lines starting with `#` are skipped
lines starting with `/` are only applied to the current directory
Status codes:
DMG: error, data damage detected
EIX: error, index damaged
old: warning, file replaced by an older version
new: new file
upd: file updated
ok : check ok
ign: ignored (see .chkbitignore)
EXC: internal exception
``` ```
chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up. chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up.

153
check/context.go Normal file
View File

@ -0,0 +1,153 @@
package check
import (
"errors"
"os"
"path/filepath"
"sync"
)
type Context struct {
NumWorkers int
Force bool
Update bool
ShowIgnoredOnly bool
HashAlgo string
SkipSymlinks bool
IndexFilename string
IgnoreFilename string
WorkQueue chan *WorkItem
LogQueue chan *LogEvent
PerfQueue chan *PerfEvent
wg sync.WaitGroup
}
func NewContext(numWorkers int, force bool, update bool, showIgnoredOnly bool, hashAlgo string, skipSymlinks bool, indexFilename string, ignoreFilename string) (*Context, error) {
if indexFilename[0] != '.' {
return nil, errors.New("The index filename must start with a dot!")
}
if ignoreFilename[0] != '.' {
return nil, errors.New("The ignore filename must start with a dot!")
}
if hashAlgo != "md5" && hashAlgo != "sha512" && hashAlgo != "blake3" {
return nil, errors.New(hashAlgo + " is unknown.")
}
return &Context{
NumWorkers: numWorkers,
Force: force,
Update: update,
ShowIgnoredOnly: showIgnoredOnly,
HashAlgo: hashAlgo,
SkipSymlinks: skipSymlinks,
IndexFilename: indexFilename,
IgnoreFilename: ignoreFilename,
WorkQueue: make(chan *WorkItem, numWorkers*10),
LogQueue: make(chan *LogEvent, numWorkers*100),
PerfQueue: make(chan *PerfEvent, numWorkers*10),
}, nil
}
func (context *Context) log(stat Status, message string) {
context.LogQueue <- &LogEvent{stat, message}
}
func (context *Context) logErr(path string, err error) {
context.LogQueue <- &LogEvent{STATUS_PANIC, path + ": " + err.Error()}
}
func (context *Context) perfMonFiles(numFiles int64) {
context.PerfQueue <- &PerfEvent{numFiles, 0}
}
func (context *Context) perfMonBytes(numBytes int64) {
context.PerfQueue <- &PerfEvent{0, numBytes}
}
func (context *Context) addWork(path string, filesToIndex []string, ignore *Ignore) {
context.WorkQueue <- &WorkItem{path, filesToIndex, ignore}
}
func (context *Context) endWork() {
context.WorkQueue <- nil
}
func (context *Context) isChkbitFile(name string) bool {
return name == context.IndexFilename || name == context.IgnoreFilename
}
func (context *Context) Start(pathList []string) {
var wg sync.WaitGroup
wg.Add(context.NumWorkers)
for i := 0; i < context.NumWorkers; i++ {
go func(id int) {
defer wg.Done()
context.RunWorker(id)
}(i)
}
go func() {
for _, path := range pathList {
context.scanDir(path, nil)
}
for i := 0; i < context.NumWorkers; i++ {
context.endWork()
}
}()
wg.Wait()
context.LogQueue <- nil
}
func (context *Context) scanDir(root string, parentIgnore *Ignore) {
files, err := os.ReadDir(root)
if err != nil {
context.logErr(root+"/", err)
return
}
isDir := func(file os.DirEntry, path string) bool {
if file.IsDir() {
return true
}
ft := file.Type()
if !context.SkipSymlinks && ft&os.ModeSymlink != 0 {
rpath, err := filepath.EvalSymlinks(path)
if err == nil {
fi, err := os.Lstat(rpath)
return err == nil && fi.IsDir()
}
}
return false
}
var dirList []string
var filesToIndex []string
for _, file := range files {
path := filepath.Join(root, file.Name())
if file.Name()[0] == '.' {
if context.ShowIgnoredOnly && !context.isChkbitFile(file.Name()) {
context.log(STATUS_IGNORE, path)
}
continue
}
if isDir(file, path) {
dirList = append(dirList, file.Name())
} else if file.Type().IsRegular() {
filesToIndex = append(filesToIndex, file.Name())
}
}
ignore, err := GetIgnore(context, root, parentIgnore)
if err != nil {
context.logErr(root+"/", err)
}
context.addWork(root, filesToIndex, ignore)
for _, name := range dirList {
if !ignore.shouldIgnore(name) {
context.scanDir(filepath.Join(root, name), ignore)
} else {
context.log(STATUS_IGNORE, name+"/")
}
}
}

57
check/hashfile.go Normal file
View File

@ -0,0 +1,57 @@
package check
import (
"crypto/md5"
"crypto/sha512"
"encoding/hex"
"errors"
"hash"
"io"
"os"
"lukechampine.com/blake3"
)
const BLOCKSIZE = 2 << 10 << 7 // kb
func Hashfile(path string, hashAlgo string, perfMonBytes func(int64)) (string, error) {
var h hash.Hash
switch hashAlgo {
case "md5":
h = md5.New()
case "sha512":
h = sha512.New()
case "blake3":
h = blake3.New(32, nil)
default:
return "", errors.New("algo '" + hashAlgo + "' is unknown.")
}
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
buf := make([]byte, BLOCKSIZE)
for {
bytesRead, err := file.Read(buf)
if err != nil && err != io.EOF {
return "", err
}
if bytesRead == 0 {
break
}
h.Write(buf[:bytesRead])
if perfMonBytes != nil {
perfMonBytes(int64(bytesRead))
}
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func HashMd5(data []byte) string {
h := md5.New()
h.Write(data)
return hex.EncodeToString(h.Sum(nil))
}

90
check/ignore.go Normal file
View File

@ -0,0 +1,90 @@
package check
import (
"bufio"
"os"
"path/filepath"
"strings"
)
type Ignore struct {
parentIgnore *Ignore
context *Context
path string
name string
itemList []string
}
func GetIgnore(context *Context, path string, parentIgnore *Ignore) (*Ignore, error) {
ignore := &Ignore{
parentIgnore: parentIgnore,
context: context,
path: path,
name: filepath.Base(path) + "/",
}
err := ignore.loadIgnore()
if err != nil {
return nil, err
}
return ignore, nil
}
func (ignore *Ignore) getIgnoreFilepath() string {
return filepath.Join(ignore.path, ignore.context.IgnoreFilename)
}
func (ignore *Ignore) loadIgnore() error {
if _, err := os.Stat(ignore.getIgnoreFilepath()); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
file, err := os.Open(ignore.getIgnoreFilepath())
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" && line[0] != '#' {
ignore.itemList = append(ignore.itemList, line)
}
}
return scanner.Err()
}
func (ignore *Ignore) shouldIgnore(name string) bool {
return ignore.shouldIgnore2(name, "")
}
func (ignore *Ignore) shouldIgnore2(name string, fullname string) bool {
for _, item := range ignore.itemList {
if item[0] == '/' {
if len(fullname) > 0 {
continue
} else {
item = item[1:]
}
}
if match, _ := filepath.Match(item, name); match {
return true
}
if fullname != "" {
if match, _ := filepath.Match(item, fullname); match {
return true
}
}
}
if ignore.parentIgnore != nil {
if fullname != "" {
return ignore.parentIgnore.shouldIgnore2(fullname, ignore.name+fullname)
} else {
return ignore.parentIgnore.shouldIgnore2(name, ignore.name+name)
}
}
return false
}

252
check/index.go Normal file
View File

@ -0,0 +1,252 @@
package check
import (
"encoding/json"
"errors"
"os"
"path/filepath"
)
const VERSION = 2 // index version
var (
algoMd5 = "md5"
)
type IdxInfo struct {
ModTime int64 `json:"mod"`
Algo *string `json:"a,omitempty"`
Hash *string `json:"h,omitempty"`
LegacyHash *string `json:"md5,omitempty"`
}
type IndexFile struct {
V int `json:"v"`
IdxRaw json.RawMessage `json:"idx"`
IdxHash string `json:"idx_hash"`
}
type IdxInfo1 struct {
ModTime int64 `json:"mod"`
Hash string `json:"md5"`
}
type IndexFile1 struct {
Data map[string]IdxInfo1 `json:"data"`
}
type Index struct {
context *Context
path string
files []string
cur map[string]IdxInfo
new map[string]IdxInfo
updates []string
modified bool
readonly bool
}
func NewIndex(context *Context, path string, files []string, readonly bool) *Index {
return &Index{
context: context,
path: path,
files: files,
cur: make(map[string]IdxInfo),
new: make(map[string]IdxInfo),
readonly: readonly,
}
}
func (i *Index) getIndexFilepath() string {
return filepath.Join(i.path, i.context.IndexFilename)
}
func (i *Index) setMod(value bool) {
i.modified = value
}
func (i *Index) logFilePanic(name string, message string) {
i.context.log(STATUS_PANIC, filepath.Join(i.path, name)+": "+message)
}
func (i *Index) logFile(stat Status, name string) {
i.context.log(stat, filepath.Join(i.path, name))
}
func (i *Index) calcHashes(ignore *Ignore) {
for _, name := range i.files {
if ignore != nil && ignore.shouldIgnore(name) {
i.logFile(STATUS_IGNORE, name)
continue
}
var err error
var info *IdxInfo
algo := i.context.HashAlgo
if val, ok := i.cur[name]; ok {
// existing
if val.LegacyHash != nil {
// convert from py1 to new format
val = IdxInfo{
ModTime: val.ModTime,
Algo: &algoMd5,
Hash: val.LegacyHash,
}
i.cur[name] = val
}
if val.Algo != nil {
algo = *val.Algo
}
info, err = i.calcFile(name, algo)
} else {
if i.readonly {
info = &IdxInfo{Algo: &algo}
} else {
info, err = i.calcFile(name, algo)
}
}
if err != nil {
i.logFilePanic(name, err.Error())
} else {
i.new[name] = *info
}
}
}
func (i *Index) showIgnoredOnly(ignore *Ignore) {
for _, name := range i.files {
if ignore.shouldIgnore(name) {
i.logFile(STATUS_IGNORE, name)
}
}
}
func (i *Index) checkFix(force bool) {
for name, b := range i.new {
if a, ok := i.cur[name]; !ok {
i.logFile(STATUS_NEW, name)
i.setMod(true)
continue
} else {
amod := int64(a.ModTime)
bmod := int64(b.ModTime)
if a.Hash != nil && b.Hash != nil && *a.Hash == *b.Hash {
i.logFile(STATUS_OK, name)
if amod != bmod {
i.setMod(true)
}
continue
}
if amod == bmod {
i.logFile(STATUS_ERR_DMG, name)
if !force {
i.new[name] = a
} else {
i.setMod(true)
}
} else if amod < bmod {
i.logFile(STATUS_UPDATE, name)
i.setMod(true)
} else if amod > bmod {
i.logFile(STATUS_WARN_OLD, name)
i.setMod(true)
}
}
}
}
func (i *Index) calcFile(name string, a string) (*IdxInfo, error) {
path := filepath.Join(i.path, name)
info, _ := os.Stat(path)
mtime := int64(info.ModTime().UnixNano() / 1e6)
h, err := Hashfile(path, a, i.context.perfMonBytes)
if err != nil {
return nil, err
}
i.context.perfMonFiles(1)
return &IdxInfo{
ModTime: mtime,
Algo: &a,
Hash: &h,
}, nil
}
func (i *Index) save() (bool, error) {
if i.modified {
if i.readonly {
return false, errors.New("Error trying to save a readonly index.")
}
text, err := json.Marshal(i.new)
if err != nil {
return false, err
}
data := IndexFile{
V: VERSION,
IdxRaw: text,
IdxHash: HashMd5(text),
}
file, err := json.Marshal(data)
if err != nil {
return false, err
}
err = os.WriteFile(i.getIndexFilepath(), file, 0644)
if err != nil {
return false, err
}
i.setMod(false)
return true, nil
} else {
return false, nil
}
}
func (i *Index) load() error {
if _, err := os.Stat(i.getIndexFilepath()); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
i.setMod(false)
file, err := os.ReadFile(i.getIndexFilepath())
if err != nil {
return err
}
var data IndexFile
err = json.Unmarshal(file, &data)
if err != nil {
return err
}
if data.IdxRaw != nil {
err = json.Unmarshal(data.IdxRaw, &i.cur)
if err != nil {
return err
}
text := data.IdxRaw
if data.IdxHash != HashMd5(text) {
// old versions may have save the JSON encoded with extra spaces
text, _ = json.Marshal(data.IdxRaw)
} else {
}
if data.IdxHash != HashMd5(text) {
i.setMod(true)
i.logFile(STATUS_ERR_IDX, i.getIndexFilepath())
}
} else {
var data1 IndexFile1
json.Unmarshal(file, &data1)
if data1.Data != nil {
// convert from js to new format
for name, item := range data1.Data {
i.cur[name] = IdxInfo{
ModTime: item.ModTime,
Algo: &algoMd5,
Hash: &item.Hash,
}
}
}
}
return nil
}

37
check/status.go Normal file
View File

@ -0,0 +1,37 @@
package check
type Status string
const (
STATUS_PANIC Status = "EXC"
STATUS_ERR_DMG Status = "DMG"
STATUS_ERR_IDX Status = "EIX"
STATUS_WARN_OLD Status = "old"
STATUS_NEW Status = "new"
STATUS_UPDATE Status = "upd"
STATUS_OK Status = "ok "
STATUS_IGNORE Status = "ign"
STATUS_UPDATE_INDEX Status = "iup"
)
func (s Status) String() string {
return (string)(s)
}
func (s Status) IsErrorOrWarning() bool {
return s == STATUS_PANIC || s == STATUS_ERR_DMG || s == STATUS_ERR_IDX || s == STATUS_WARN_OLD
}
func (s Status) IsVerbose() bool {
return s == STATUS_OK || s == STATUS_IGNORE
}
type LogEvent struct {
Stat Status
Message string
}
type PerfEvent struct {
NumFiles int64
NumBytes int64
}

37
check/worker.go Normal file
View File

@ -0,0 +1,37 @@
package check
type WorkItem struct {
path string
filesToIndex []string
ignore *Ignore
}
func (context *Context) RunWorker(id int) {
for {
item := <-context.WorkQueue
if item == nil {
break
}
index := NewIndex(context, item.path, item.filesToIndex, !context.Update)
err := index.load()
if err != nil {
context.log(STATUS_PANIC, index.getIndexFilepath()+": "+err.Error())
}
if context.ShowIgnoredOnly {
index.showIgnoredOnly(item.ignore)
} else {
index.calcHashes(item.ignore)
index.checkFix(context.Force)
if context.Update {
if changed, err := index.save(); err != nil {
context.logErr(item.path, err)
} else if changed {
context.log(STATUS_UPDATE_INDEX, "")
}
}
}
}
}

24
cmd/chkbit/help.go Normal file
View File

@ -0,0 +1,24 @@
package main
var headerHelp = `Checks the data integrity of your files.
For help tips run "chkbit -H" or go to
https://github.com/laktak/chkbit
`
var helpTips = `
.chkbitignore rules:
each line should contain exactly one name
you may use Unix shell-style wildcards (see README)
lines starting with '#' are skipped
lines starting with '/' are only applied to the current directory
Status codes:
DMG: error, data damage detected
EIX: error, index damaged
old: warning, file replaced by an older version
new: new file
upd: file updated
ok : check ok
ign: ignored (see .chkbitignore)
EXC: exception/panic
`

335
cmd/chkbit/main.go Normal file
View File

@ -0,0 +1,335 @@
package main
import (
"fmt"
"io"
"log"
"os"
"strings"
"sync"
"time"
"github.com/alecthomas/kong"
"github.com/laktak/chkbit/check"
"github.com/laktak/chkbit/term"
"github.com/laktak/chkbit/util"
)
type Progress int
const (
Quiet Progress = iota
Summary
Plain
Fancy
)
const (
updateInterval = time.Millisecond * 300
sizeMB int64 = 1024 * 1024
)
var appVersion = "vdev"
var (
termBG = term.Bg8(240)
termSep = "|"
termSepFG = term.Fg8(235)
termFG1 = term.Fg8(255)
termFG2 = term.Fg8(228)
termFG3 = term.Fg8(202)
termOKFG = term.Fg4(2)
termAlertFG = term.Fg4(1)
)
var cli struct {
Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"`
Tips bool `short:"H" help:"Show tips."`
Update bool `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"`
ShowIgnoredOnly bool `help:"only show ignored files"`
Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
Force bool `short:"f" help:"force update of damaged items"`
SkipSymlinks bool `short:"s" help:"do not follow symlinks"`
LogFile string `short:"l" help:"write to a logfile if specified"`
LogVerbose bool `help:"verbose logging"`
IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
IgnoreName string `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"`
Workers int `short:"w" default:"5" help:"number of workers to use (default: 5)"`
Plain bool `help:"show plain status instead of being fancy"`
Quiet bool `short:"q" help:"quiet, don't show progress/information"`
Verbose bool `short:"v" help:"verbose output"`
Version bool `short:"V" help:"show version information"`
}
type Main struct {
dmgList []string
errList []string
numIdxUpd int
numNew int
numUpd int
verbose bool
logger *log.Logger
logVerbose bool
progress Progress
total int
termWidth int
fps *RateCalc
bps *RateCalc
}
func (m *Main) log(text string) {
m.logger.Println(time.Now().UTC().Format("2006-01-02 15:04:05"), text)
}
func (m *Main) logStatus(stat check.Status, path string) {
if stat == check.STATUS_UPDATE_INDEX {
m.numIdxUpd++
} else {
if stat == check.STATUS_ERR_DMG {
m.total++
m.dmgList = append(m.dmgList, path)
} else if stat == check.STATUS_PANIC {
m.errList = append(m.errList, path)
} else if stat == check.STATUS_OK || stat == check.STATUS_UPDATE || stat == check.STATUS_NEW {
m.total++
if stat == check.STATUS_UPDATE {
m.numUpd++
} else if stat == check.STATUS_NEW {
m.numNew++
}
}
if m.logVerbose || stat != check.STATUS_OK && stat != check.STATUS_IGNORE {
m.log(stat.String() + " " + path)
}
if m.verbose || !stat.IsVerbose() {
col := ""
if stat.IsErrorOrWarning() {
col = termAlertFG
}
term.Printline(col, stat.String(), " ", path, term.Reset)
}
}
}
func (m *Main) showStatus(context *check.Context) {
last := time.Now().Add(-updateInterval)
stat := ""
for {
select {
case item := <-context.LogQueue:
if item == nil {
if m.progress == Fancy {
term.Printline("")
}
return
}
m.logStatus(item.Stat, item.Message)
if m.progress == Fancy {
term.Write(termBG, termFG1, stat, term.ClearLine(0), term.Reset, "\r")
} else {
fmt.Print(m.total, "\r")
}
case perf := <-context.PerfQueue:
now := time.Now()
m.fps.Push(now, perf.NumFiles)
m.bps.Push(now, perf.NumBytes)
if last.Add(updateInterval).Before(now) {
last = now
if m.progress == Fancy {
statF := fmt.Sprintf("%d files/s", m.fps.Last())
statB := fmt.Sprintf("%d MB/s", m.bps.Last()/sizeMB)
stat = "RW"
if !context.Update {
stat = "RO"
}
stat = fmt.Sprintf("[%s:%d] %5d files $ %s %-13s $ %s %-13s",
stat, context.NumWorkers, m.total,
util.Sparkline(m.fps.Stats), statF,
util.Sparkline(m.bps.Stats), statB)
stat = util.LeftTruncate(stat, m.termWidth-1)
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG2, 1)
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG3, 1)
term.Write(termBG, termFG1, stat, term.ClearLine(0), term.Reset, "\r")
} else if m.progress == Plain {
fmt.Print(m.total, "\r")
}
}
}
}
}
func (m *Main) process() *check.Context {
if cli.Update && cli.ShowIgnoredOnly {
fmt.Println("Error: use either --update or --show-ignored-only!")
return nil
}
context, err := check.NewContext(cli.Workers, cli.Force, cli.Update, cli.ShowIgnoredOnly, cli.Algo, cli.SkipSymlinks, cli.IndexName, cli.IgnoreName)
if err != nil {
fmt.Println(err)
return nil
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
m.showStatus(context)
}()
context.Start(cli.Paths)
wg.Wait()
return context
}
func (m *Main) printResult(context *check.Context) {
cprint := func(col, text string) {
if m.progress != Quiet {
if m.progress == Fancy {
term.Printline(col, text, term.Reset)
} else {
fmt.Println(text)
}
}
}
eprint := func(col, text string) {
if m.progress == Fancy {
term.Write(col)
fmt.Fprintln(os.Stderr, text)
term.Write(term.Reset)
} else {
fmt.Fprintln(os.Stderr, text)
}
}
if m.progress != Quiet {
mode := ""
if !context.Update {
mode = " in readonly mode"
}
status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(m.total, "file"), mode)
cprint(termOKFG, status)
m.log(status)
if m.progress == Fancy && m.total > 0 {
elapsed := time.Since(m.fps.Start)
elapsedS := elapsed.Seconds()
fmt.Println("-", elapsed.Truncate(time.Second), "elapsed")
fmt.Printf("- %.2f files/second\n", (float64(m.fps.Total)+float64(m.fps.Current))/elapsedS)
fmt.Printf("- %.2f MB/second\n", (float64(m.bps.Total)+float64(m.bps.Current))/float64(sizeMB)/elapsedS)
}
if context.Update {
if m.numIdxUpd > 0 {
cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated",
util.LangNum1Choice(m.numIdxUpd, "directory was", "directories were"),
util.LangNum1Choice(m.numNew, "file hash was", "file hashes were"),
util.LangNum1Choice(m.numUpd, "file hash was", "file hashes were")))
}
} else if m.numNew+m.numUpd > 0 {
cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added and\n- %s would have been updated.",
util.LangNum1MutateSuffix(m.numNew, "file"),
util.LangNum1MutateSuffix(m.numUpd, "file")))
}
}
if len(m.dmgList) > 0 {
eprint(termAlertFG, "chkbit detected damage in these files:")
for _, err := range m.dmgList {
fmt.Fprintln(os.Stderr, err)
}
n := len(m.dmgList)
status := fmt.Sprintf("error: detected %s with damage!", util.LangNum1MutateSuffix(n, "file"))
m.log(status)
eprint(termAlertFG, status)
}
if len(m.errList) > 0 {
status := "chkbit ran into errors"
m.log(status + "!")
eprint(termAlertFG, status+":")
for _, err := range m.errList {
fmt.Fprintln(os.Stderr, err)
}
}
if len(m.dmgList) > 0 || len(m.errList) > 0 {
os.Exit(1)
}
}
func (m *Main) run() {
if len(os.Args) < 2 {
os.Args = append(os.Args, "--help")
}
kong.Parse(&cli,
kong.Name("chkbit"),
kong.Description(""),
kong.UsageOnError(),
)
if cli.Tips {
fmt.Println(helpTips)
os.Exit(0)
}
if cli.Version {
fmt.Println("github.com/laktak/chkbit")
fmt.Println(appVersion)
return
}
m.verbose = cli.Verbose || cli.ShowIgnoredOnly
if cli.LogFile != "" {
m.logVerbose = cli.LogVerbose
f, err := os.OpenFile(cli.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
fmt.Println(err)
return
}
defer f.Close()
m.logger = log.New(f, "", 0)
}
if cli.Quiet {
m.progress = Quiet
} else if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
m.progress = Summary
} else if cli.Plain {
m.progress = Plain
} else {
m.progress = Fancy
}
if len(cli.Paths) > 0 {
m.log("chkbit " + strings.Join(cli.Paths, ", "))
context := m.process()
if context != nil && !context.ShowIgnoredOnly {
m.printResult(context)
}
} else {
fmt.Println("specify a path to check, see -h")
}
}
func main() {
defer func() {
if r := recover(); r != nil {
fmt.Println(r)
os.Exit(1)
}
}()
termWidth := term.GetWidth()
m := &Main{
logger: log.New(io.Discard, "", 0),
termWidth: termWidth,
fps: NewRateCalc(time.Second, (termWidth-70)/2),
bps: NewRateCalc(time.Second, (termWidth-70)/2),
}
m.run()
}

52
cmd/chkbit/rate_calc.go Normal file
View File

@ -0,0 +1,52 @@
package main
import (
"time"
)
type RateCalc struct {
Interval time.Duration
MaxStat int
Start time.Time
Updated time.Time
Total int64
Current int64
Stats []int64
}
func NewRateCalc(interval time.Duration, maxStat int) *RateCalc {
if maxStat < 10 {
maxStat = 10
}
rc := &RateCalc{
Interval: interval,
MaxStat: maxStat,
}
rc.Reset()
return rc
}
func (rc *RateCalc) Reset() {
rc.Start = time.Now()
rc.Updated = rc.Start
rc.Total = 0
rc.Current = 0
rc.Stats = make([]int64, rc.MaxStat)
}
func (rc *RateCalc) Last() int64 {
return rc.Stats[len(rc.Stats)-1]
}
func (rc *RateCalc) Push(ts time.Time, value int64) {
for rc.Updated.Add(rc.Interval).Before(ts) {
rc.Stats = append(rc.Stats, rc.Current)
if len(rc.Stats) > rc.MaxStat {
rc.Stats = rc.Stats[len(rc.Stats)-rc.MaxStat:]
}
rc.Total += rc.Current
rc.Current = 0
rc.Updated = rc.Updated.Add(rc.Interval)
}
rc.Current += value
}

14
go.mod Normal file
View File

@ -0,0 +1,14 @@
module github.com/laktak/chkbit
go 1.22.3
require (
github.com/alecthomas/kong v0.9.0
golang.org/x/sys v0.23.0
lukechampine.com/blake3 v1.3.0
)
require (
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
golang.org/x/term v0.23.0 // indirect
)

18
go.sum Normal file
View File

@ -0,0 +1,18 @@
github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA=
github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE=
lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=

8
scripts/build Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -eE -o pipefail
script_dir=$(dirname "$(realpath "$0")")
cd $script_dir/..
version=$(git describe --tags --always)
go build -ldflags="-X main.appVersion=$version" ./cmd/chkbit

13
scripts/chkfmt Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
set -eE -o pipefail
script_dir=$(dirname "$(realpath "$0")")
cd $script_dir/..
res="$(gofmt -l . 2>&1)"
if [ -n "$res" ]; then
echo "gofmt check failed:"
echo "${res}"
exit 1
fi

7
scripts/lint Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -eE -o pipefail
script_dir=$(dirname "$(realpath "$0")")
cd $script_dir/..
go vet -structtag=false -composites=false ./...

13
scripts/run_test_prep Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
export TZ='UTC'
root="/tmp/chkbit"
go run scripts/run_test_prep.go
cd $root/root
mv $root/root/people $root/people
ln -s ../people people
ln -s ../../people/face/office-door.pdf day/friend/office-door.pdf
find -L | wc -l

89
scripts/run_test_prep.go Normal file
View File

@ -0,0 +1,89 @@
package main
import (
"fmt"
"os"
"path/filepath"
"time"
)
var (
startList = []string{"time", "year", "people", "way", "day", "thing"}
wordList = []string{"life", "world", "school", "state", "family", "student", "group", "country", "problem", "hand", "part", "place", "case", "week", "company", "system", "program", "work", "government", "number", "night", "point", "home", "water", "room", "mother", "area", "money", "story", "fact", "month", "lot", "right", "study", "book", "eye", "job", "word", "business", "issue", "side", "kind", "head", "house", "service", "friend", "father", "power", "hour", "game", "line", "end", "member", "law", "car", "city", "community", "name", "president", "team", "minute", "idea", "kid", "body", "information", "back", "face", "others", "level", "office", "door", "health", "person", "art", "war", "history", "party", "result", "change", "morning", "reason", "research", "moment", "air", "teacher", "force", "education"}
extList = []string{"txt", "md", "pdf", "jpg", "jpeg", "png", "mp4", "mp3", "csv"}
startDate = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
endDate = time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC)
dateList = []time.Time{}
wordIdx = 0
extIdx = 0
dateIdx = 0
)
func nextWord() string {
word := wordList[wordIdx%len(wordList)]
wordIdx++
return word
}
func nextExt() string {
ext := extList[extIdx%len(extList)]
extIdx++
return ext
}
func setDate(filename string, r int) {
date := dateList[dateIdx%len(dateList)]
m := 17 * dateIdx / len(dateList)
date = date.Add(time.Duration(m) * time.Hour)
dateIdx++
os.Chtimes(filename, date, date)
}
func genFile(dir string, a int) {
os.MkdirAll(dir, 0755)
for i := 1; i <= 5; i++ {
size := a*i*wordIdx*100 + extIdx
file := nextWord() + "-" + nextWord()
if i%3 == 0 {
file += "-" + nextWord()
}
file += "." + nextExt()
path := filepath.Join(dir, file)
os.WriteFile(path, make([]byte, size), 0644)
setDate(path, size*size)
}
}
func genDir(root string) {
for _, start := range startList {
for i := 1; i <= 5; i++ {
dir := filepath.Join(root, start, nextWord())
genFile(dir, 1)
if wordIdx%3 == 0 {
dir = filepath.Join(dir, nextWord())
genFile(dir, 1)
}
}
}
}
func main() {
root := "/tmp/chkbit"
var c int64 = 50
interval := (int64)(endDate.Sub(startDate).Seconds()) / c
for i := range make([]int64, c) {
dateList = append(dateList, startDate.Add(time.Duration(interval*(int64)(i))*time.Second))
}
if err := os.RemoveAll(root); err == nil {
genDir(filepath.Join(root, "root"))
fmt.Println("Ready.")
} else {
fmt.Println("Failed to clean")
}
}

19
scripts/run_tests Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -e
export TZ='UTC'
script_dir=$(dirname "$(realpath "$0")")
base_dir=$(dirname "$script_dir")
#dir=$(realpath "$script_dir/../testdata/run_test")
root="/tmp/chkbit/root"
if [[ ! -d $root ]]; then
echo "must run run_test_prep first"
exit 1
fi
# setup
$script_dir/build
"$base_dir/chkbit" -u /tmp/chkbit

7
scripts/tests Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -e
script_dir=$(dirname "$(realpath "$0")")
cd $script_dir/..
go test -v ./util

55
scripts/xbuild Executable file
View File

@ -0,0 +1,55 @@
#!/bin/bash
set -eE -o pipefail
script_dir=$(dirname "$(realpath "$0")")
cd $script_dir/..
if [ -z "$version" ]; then
version=$(git rev-parse HEAD)
fi
echo "building version $version"
mkdir -p dist
rm -f dist/*
build() {
echo "- $1-$2"
rm -f dist/chkbit
CGO_ENABLED=0 GOOS="$1" GOARCH="$2" go build -o dist -ldflags="-X main.appVersion=$version" ./cmd/chkbit
pushd dist
case "$1" in
windows)
outfile="chkbit-$1-$2.zip"
zip "$outfile" chkbit.exe --move
;;
*)
outfile="chkbit-$1-$2.tar.gz"
tar -czf "$outfile" chkbit --remove-files
;;
esac
popd
}
if [[ -z $2 ]]; then
build android arm64
build darwin amd64
build darwin arm64
build freebsd amd64
build freebsd arm64
build freebsd riscv64
build linux amd64
build linux arm64
build linux riscv64
build netbsd amd64
build netbsd arm64
build openbsd amd64
build openbsd arm64
build windows amd64
build windows arm64
else
build $1 $2
fi

82
term/term.go Normal file
View File

@ -0,0 +1,82 @@
package term
import (
"fmt"
"os"
"golang.org/x/term"
)
var (
isTerm = false
noColor = false
stdoutFd = 0
)
func init() {
stdoutFd = int(os.Stdout.Fd())
isTerm = term.IsTerminal(stdoutFd)
if isTerm {
noColor = os.Getenv("NO_COLOR") != ""
} else {
noColor = true
}
}
const (
Reset = "\033[0m"
Bold = "\033[01m"
Disable = "\033[02m"
Underline = "\033[04m"
Reverse = "\033[07m"
Strikethrough = "\033[09m"
Invisible = "\033[08m"
)
func Write(text ...interface{}) {
fmt.Print(text...)
}
func Printline(text ...interface{}) {
fmt.Print(text...)
fmt.Println(ClearLine(0))
}
func Fg4(col int) string {
if noColor {
return ""
}
if col < 8 {
return fmt.Sprintf("\033[%dm", 30+col)
}
return fmt.Sprintf("\033[%dm", 90-8+col)
}
func Fg8(col int) string {
if noColor {
return ""
}
return fmt.Sprintf("\033[38;5;%dm", col)
}
func Bg8(col int) string {
if noColor {
return ""
}
return fmt.Sprintf("\033[48;5;%dm", col)
}
func ClearLine(opt int) string {
// 0=to end, 1=from start, 2=all
return fmt.Sprintf("\033[%dK", opt)
}
func GetWidth() int {
if isTerm {
width, _, err := term.GetSize(stdoutFd)
if err == nil {
return width
}
}
return 80
}

21
term/term_windows.go Normal file
View File

@ -0,0 +1,21 @@
package term
import (
"os"
"golang.org/x/sys/windows"
)
// from https://github.com/fatih/color
func init() {
// Opt-in for ansi color support for current process.
// https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences
var outMode uint32
out := windows.Handle(os.Stdout.Fd())
if err := windows.GetConsoleMode(out, &outMode); err != nil {
return
}
outMode |= windows.ENABLE_PROCESSED_OUTPUT | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING
_ = windows.SetConsoleMode(out, outMode)
}

35
util/fm.go Normal file
View File

@ -0,0 +1,35 @@
package util
import (
"math"
)
func Minimum(series []int64) int64 {
var min int64 = math.MaxInt64
for _, value := range series {
if value < min {
min = value
}
}
return min
}
func Maximum(series []int64) int64 {
var max int64 = math.MinInt64
for _, value := range series {
if value > max {
max = value
}
}
return max
}
func Clamp(min int64, max int64, n int64) int64 {
if n < min {
return min
}
if n > max {
return max
}
return n
}

19
util/lang.go Normal file
View File

@ -0,0 +1,19 @@
package util
import "fmt"
func LangNum1MutateSuffix(num int, u string) string {
s := ""
if num != 1 {
s = "s"
}
return fmt.Sprintf("%d %s%s", num, u, s)
}
func LangNum1Choice(num int, u1, u2 string) string {
u := u1
if num != 1 {
u = u2
}
return fmt.Sprintf("%d %s", num, u)
}

32
util/sparkline.go Normal file
View File

@ -0,0 +1,32 @@
package util
import (
"math"
)
var sparkChars = []rune{'▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'}
func Sparkline(series []int64) string {
out := make([]rune, len(series))
min := Minimum(series)
max := Maximum(series)
dataRange := max - min
if dataRange == 0 {
for i := range series {
out[i] = sparkChars[0]
}
} else {
step := float64(len(sparkChars)-1) / float64(dataRange)
for i, n := range series {
idx := int(math.Round(float64(Clamp(min, max, n)-min) * step))
if idx < 0 {
out[i] = ' '
} else if idx > len(sparkChars) {
out[i] = sparkChars[len(sparkChars)-1]
} else {
out[i] = sparkChars[idx]
}
}
}
return string(out)
}

13
util/sparkline_test.go Normal file
View File

@ -0,0 +1,13 @@
package util
import (
"testing"
)
func TestSpark(t *testing.T) {
expected := "▁▁▂▄▅▇██▆▄▂"
actual := Sparkline([]int64{5, 12, 35, 73, 80, 125, 150, 142, 118, 61, 19})
if expected != actual {
t.Error("expected:", expected, "actual:", actual)
}
}

11
util/strings.go Normal file
View File

@ -0,0 +1,11 @@
package util
func LeftTruncate(s string, nMax int) string {
for i := range s {
nMax--
if nMax < 0 {
return s[:i]
}
}
return s
}

13
util/strings_test.go Normal file
View File

@ -0,0 +1,13 @@
package util
import (
"testing"
)
func TestTrunc(t *testing.T) {
expected := "ab©def"
actual := LeftTruncate(expected+"ghijk", 6)
if expected != actual {
t.Error("expected:", expected, "actual:", actual)
}
}