initial go version
This commit is contained in:
parent
bb33b02b68
commit
9e6a42f626
3
.gitignore
vendored
3
.gitignore
vendored
@ -0,0 +1,3 @@
|
||||
# bin
|
||||
/chkbit
|
||||
dist
|
59
README.md
59
README.md
@ -31,7 +31,7 @@ Remember to always maintain multiple backups for comprehensive data protection.
|
||||
```
|
||||
brew install chkbit
|
||||
```
|
||||
- Download for [Linux, macOS or Windows](https://github.com/laktak/chkbit-py/releases).
|
||||
- Download for [Linux, macOS or Windows](https://github.com/laktak/chkbit/releases).
|
||||
|
||||
|
||||
## Usage
|
||||
@ -47,45 +47,28 @@ chkbit will
|
||||
Run `chkbit PATH` to verify only.
|
||||
|
||||
```
|
||||
usage: chkbit [-h] [-u] [--show-ignored-only] [--algo ALGO] [-f] [-s] [-l FILE] [--log-verbose] [--index-name NAME] [--ignore-name NAME] [-w N] [--plain] [-q] [-v] [PATH ...]
|
||||
Usage: chkbit [<paths> ...] [flags]
|
||||
|
||||
Checks the data integrity of your files. See https://github.com/laktak/chkbit-py
|
||||
Arguments:
|
||||
[<paths> ...] directories to check
|
||||
|
||||
positional arguments:
|
||||
PATH directories to check
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-u, --update update indices (without this chkbit will verify files in readonly mode)
|
||||
--show-ignored-only only show ignored files
|
||||
--algo ALGO hash algorithm: md5, sha512, blake3 (default: blake3)
|
||||
-f, --force force update of damaged items
|
||||
-s, --skip-symlinks do not follow symlinks
|
||||
-l FILE, --log-file FILE
|
||||
write to a logfile if specified
|
||||
--log-verbose verbose logging
|
||||
--index-name NAME filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
|
||||
--ignore-name NAME filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)
|
||||
-w N, --workers N number of workers to use (default: 5)
|
||||
--plain show plain status instead of being fancy
|
||||
-q, --quiet quiet, don't show progress/information
|
||||
-v, --verbose verbose output
|
||||
|
||||
.chkbitignore rules:
|
||||
each line should contain exactly one name
|
||||
you may use Unix shell-style wildcards (see README)
|
||||
lines starting with `#` are skipped
|
||||
lines starting with `/` are only applied to the current directory
|
||||
|
||||
Status codes:
|
||||
DMG: error, data damage detected
|
||||
EIX: error, index damaged
|
||||
old: warning, file replaced by an older version
|
||||
new: new file
|
||||
upd: file updated
|
||||
ok : check ok
|
||||
ign: ignored (see .chkbitignore)
|
||||
EXC: internal exception
|
||||
Flags:
|
||||
-h, --help Show context-sensitive help.
|
||||
-H, --tips Show tips.
|
||||
-u, --update update indices (without this chkbit will verify files in readonly mode)
|
||||
--show-ignored-only only show ignored files
|
||||
--algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3)
|
||||
-f, --force force update of damaged items
|
||||
-s, --skip-symlinks do not follow symlinks
|
||||
-l, --log-file=STRING write to a logfile if specified
|
||||
--log-verbose verbose logging
|
||||
--index-name=".chkbit" filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
|
||||
--ignore-name=".chkbitignore" filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)
|
||||
-w, --workers=5 number of workers to use (default: 5)
|
||||
--plain show plain status instead of being fancy
|
||||
-q, --quiet quiet, don't show progress/information
|
||||
-v, --verbose verbose output
|
||||
-V, --version show version information
|
||||
```
|
||||
|
||||
chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up.
|
||||
|
153
check/context.go
Normal file
153
check/context.go
Normal file
@ -0,0 +1,153 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Context struct {
|
||||
NumWorkers int
|
||||
Force bool
|
||||
Update bool
|
||||
ShowIgnoredOnly bool
|
||||
HashAlgo string
|
||||
SkipSymlinks bool
|
||||
IndexFilename string
|
||||
IgnoreFilename string
|
||||
WorkQueue chan *WorkItem
|
||||
LogQueue chan *LogEvent
|
||||
PerfQueue chan *PerfEvent
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func NewContext(numWorkers int, force bool, update bool, showIgnoredOnly bool, hashAlgo string, skipSymlinks bool, indexFilename string, ignoreFilename string) (*Context, error) {
|
||||
if indexFilename[0] != '.' {
|
||||
return nil, errors.New("The index filename must start with a dot!")
|
||||
}
|
||||
if ignoreFilename[0] != '.' {
|
||||
return nil, errors.New("The ignore filename must start with a dot!")
|
||||
}
|
||||
if hashAlgo != "md5" && hashAlgo != "sha512" && hashAlgo != "blake3" {
|
||||
return nil, errors.New(hashAlgo + " is unknown.")
|
||||
}
|
||||
return &Context{
|
||||
NumWorkers: numWorkers,
|
||||
Force: force,
|
||||
Update: update,
|
||||
ShowIgnoredOnly: showIgnoredOnly,
|
||||
HashAlgo: hashAlgo,
|
||||
SkipSymlinks: skipSymlinks,
|
||||
IndexFilename: indexFilename,
|
||||
IgnoreFilename: ignoreFilename,
|
||||
WorkQueue: make(chan *WorkItem, numWorkers*10),
|
||||
LogQueue: make(chan *LogEvent, numWorkers*100),
|
||||
PerfQueue: make(chan *PerfEvent, numWorkers*10),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (context *Context) log(stat Status, message string) {
|
||||
context.LogQueue <- &LogEvent{stat, message}
|
||||
}
|
||||
|
||||
func (context *Context) logErr(path string, err error) {
|
||||
context.LogQueue <- &LogEvent{STATUS_PANIC, path + ": " + err.Error()}
|
||||
}
|
||||
|
||||
func (context *Context) perfMonFiles(numFiles int64) {
|
||||
context.PerfQueue <- &PerfEvent{numFiles, 0}
|
||||
}
|
||||
|
||||
func (context *Context) perfMonBytes(numBytes int64) {
|
||||
context.PerfQueue <- &PerfEvent{0, numBytes}
|
||||
}
|
||||
|
||||
func (context *Context) addWork(path string, filesToIndex []string, ignore *Ignore) {
|
||||
context.WorkQueue <- &WorkItem{path, filesToIndex, ignore}
|
||||
}
|
||||
|
||||
func (context *Context) endWork() {
|
||||
context.WorkQueue <- nil
|
||||
}
|
||||
|
||||
func (context *Context) isChkbitFile(name string) bool {
|
||||
return name == context.IndexFilename || name == context.IgnoreFilename
|
||||
}
|
||||
|
||||
func (context *Context) Start(pathList []string) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(context.NumWorkers)
|
||||
for i := 0; i < context.NumWorkers; i++ {
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
context.RunWorker(id)
|
||||
}(i)
|
||||
}
|
||||
go func() {
|
||||
for _, path := range pathList {
|
||||
context.scanDir(path, nil)
|
||||
}
|
||||
for i := 0; i < context.NumWorkers; i++ {
|
||||
context.endWork()
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
context.LogQueue <- nil
|
||||
}
|
||||
|
||||
func (context *Context) scanDir(root string, parentIgnore *Ignore) {
|
||||
files, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
context.logErr(root+"/", err)
|
||||
return
|
||||
}
|
||||
|
||||
isDir := func(file os.DirEntry, path string) bool {
|
||||
if file.IsDir() {
|
||||
return true
|
||||
}
|
||||
ft := file.Type()
|
||||
if !context.SkipSymlinks && ft&os.ModeSymlink != 0 {
|
||||
rpath, err := filepath.EvalSymlinks(path)
|
||||
if err == nil {
|
||||
fi, err := os.Lstat(rpath)
|
||||
return err == nil && fi.IsDir()
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var dirList []string
|
||||
var filesToIndex []string
|
||||
|
||||
for _, file := range files {
|
||||
path := filepath.Join(root, file.Name())
|
||||
if file.Name()[0] == '.' {
|
||||
if context.ShowIgnoredOnly && !context.isChkbitFile(file.Name()) {
|
||||
context.log(STATUS_IGNORE, path)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isDir(file, path) {
|
||||
dirList = append(dirList, file.Name())
|
||||
} else if file.Type().IsRegular() {
|
||||
filesToIndex = append(filesToIndex, file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
ignore, err := GetIgnore(context, root, parentIgnore)
|
||||
if err != nil {
|
||||
context.logErr(root+"/", err)
|
||||
}
|
||||
|
||||
context.addWork(root, filesToIndex, ignore)
|
||||
|
||||
for _, name := range dirList {
|
||||
if !ignore.shouldIgnore(name) {
|
||||
context.scanDir(filepath.Join(root, name), ignore)
|
||||
} else {
|
||||
context.log(STATUS_IGNORE, name+"/")
|
||||
}
|
||||
}
|
||||
}
|
57
check/hashfile.go
Normal file
57
check/hashfile.go
Normal file
@ -0,0 +1,57 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"lukechampine.com/blake3"
|
||||
)
|
||||
|
||||
const BLOCKSIZE = 2 << 10 << 7 // kb
|
||||
|
||||
func Hashfile(path string, hashAlgo string, perfMonBytes func(int64)) (string, error) {
|
||||
var h hash.Hash
|
||||
switch hashAlgo {
|
||||
case "md5":
|
||||
h = md5.New()
|
||||
case "sha512":
|
||||
h = sha512.New()
|
||||
case "blake3":
|
||||
h = blake3.New(32, nil)
|
||||
default:
|
||||
return "", errors.New("algo '" + hashAlgo + "' is unknown.")
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
buf := make([]byte, BLOCKSIZE)
|
||||
for {
|
||||
bytesRead, err := file.Read(buf)
|
||||
if err != nil && err != io.EOF {
|
||||
return "", err
|
||||
}
|
||||
if bytesRead == 0 {
|
||||
break
|
||||
}
|
||||
h.Write(buf[:bytesRead])
|
||||
if perfMonBytes != nil {
|
||||
perfMonBytes(int64(bytesRead))
|
||||
}
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func HashMd5(data []byte) string {
|
||||
h := md5.New()
|
||||
h.Write(data)
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
90
check/ignore.go
Normal file
90
check/ignore.go
Normal file
@ -0,0 +1,90 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Ignore struct {
|
||||
parentIgnore *Ignore
|
||||
context *Context
|
||||
path string
|
||||
name string
|
||||
itemList []string
|
||||
}
|
||||
|
||||
func GetIgnore(context *Context, path string, parentIgnore *Ignore) (*Ignore, error) {
|
||||
ignore := &Ignore{
|
||||
parentIgnore: parentIgnore,
|
||||
context: context,
|
||||
path: path,
|
||||
name: filepath.Base(path) + "/",
|
||||
}
|
||||
err := ignore.loadIgnore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ignore, nil
|
||||
}
|
||||
|
||||
func (ignore *Ignore) getIgnoreFilepath() string {
|
||||
return filepath.Join(ignore.path, ignore.context.IgnoreFilename)
|
||||
}
|
||||
|
||||
func (ignore *Ignore) loadIgnore() error {
|
||||
if _, err := os.Stat(ignore.getIgnoreFilepath()); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(ignore.getIgnoreFilepath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line != "" && line[0] != '#' {
|
||||
ignore.itemList = append(ignore.itemList, line)
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func (ignore *Ignore) shouldIgnore(name string) bool {
|
||||
return ignore.shouldIgnore2(name, "")
|
||||
}
|
||||
|
||||
func (ignore *Ignore) shouldIgnore2(name string, fullname string) bool {
|
||||
for _, item := range ignore.itemList {
|
||||
if item[0] == '/' {
|
||||
if len(fullname) > 0 {
|
||||
continue
|
||||
} else {
|
||||
item = item[1:]
|
||||
}
|
||||
}
|
||||
if match, _ := filepath.Match(item, name); match {
|
||||
return true
|
||||
}
|
||||
if fullname != "" {
|
||||
if match, _ := filepath.Match(item, fullname); match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
if ignore.parentIgnore != nil {
|
||||
if fullname != "" {
|
||||
return ignore.parentIgnore.shouldIgnore2(fullname, ignore.name+fullname)
|
||||
} else {
|
||||
return ignore.parentIgnore.shouldIgnore2(name, ignore.name+name)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
252
check/index.go
Normal file
252
check/index.go
Normal file
@ -0,0 +1,252 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const VERSION = 2 // index version
|
||||
var (
|
||||
algoMd5 = "md5"
|
||||
)
|
||||
|
||||
type IdxInfo struct {
|
||||
ModTime int64 `json:"mod"`
|
||||
Algo *string `json:"a,omitempty"`
|
||||
Hash *string `json:"h,omitempty"`
|
||||
LegacyHash *string `json:"md5,omitempty"`
|
||||
}
|
||||
|
||||
type IndexFile struct {
|
||||
V int `json:"v"`
|
||||
IdxRaw json.RawMessage `json:"idx"`
|
||||
IdxHash string `json:"idx_hash"`
|
||||
}
|
||||
|
||||
type IdxInfo1 struct {
|
||||
ModTime int64 `json:"mod"`
|
||||
Hash string `json:"md5"`
|
||||
}
|
||||
|
||||
type IndexFile1 struct {
|
||||
Data map[string]IdxInfo1 `json:"data"`
|
||||
}
|
||||
|
||||
type Index struct {
|
||||
context *Context
|
||||
path string
|
||||
files []string
|
||||
cur map[string]IdxInfo
|
||||
new map[string]IdxInfo
|
||||
updates []string
|
||||
modified bool
|
||||
readonly bool
|
||||
}
|
||||
|
||||
func NewIndex(context *Context, path string, files []string, readonly bool) *Index {
|
||||
return &Index{
|
||||
context: context,
|
||||
path: path,
|
||||
files: files,
|
||||
cur: make(map[string]IdxInfo),
|
||||
new: make(map[string]IdxInfo),
|
||||
readonly: readonly,
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Index) getIndexFilepath() string {
|
||||
return filepath.Join(i.path, i.context.IndexFilename)
|
||||
}
|
||||
|
||||
func (i *Index) setMod(value bool) {
|
||||
i.modified = value
|
||||
}
|
||||
|
||||
func (i *Index) logFilePanic(name string, message string) {
|
||||
i.context.log(STATUS_PANIC, filepath.Join(i.path, name)+": "+message)
|
||||
}
|
||||
|
||||
func (i *Index) logFile(stat Status, name string) {
|
||||
i.context.log(stat, filepath.Join(i.path, name))
|
||||
}
|
||||
|
||||
func (i *Index) calcHashes(ignore *Ignore) {
|
||||
for _, name := range i.files {
|
||||
if ignore != nil && ignore.shouldIgnore(name) {
|
||||
i.logFile(STATUS_IGNORE, name)
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
var info *IdxInfo
|
||||
algo := i.context.HashAlgo
|
||||
if val, ok := i.cur[name]; ok {
|
||||
// existing
|
||||
if val.LegacyHash != nil {
|
||||
// convert from py1 to new format
|
||||
val = IdxInfo{
|
||||
ModTime: val.ModTime,
|
||||
Algo: &algoMd5,
|
||||
Hash: val.LegacyHash,
|
||||
}
|
||||
i.cur[name] = val
|
||||
}
|
||||
if val.Algo != nil {
|
||||
algo = *val.Algo
|
||||
}
|
||||
info, err = i.calcFile(name, algo)
|
||||
} else {
|
||||
if i.readonly {
|
||||
info = &IdxInfo{Algo: &algo}
|
||||
} else {
|
||||
info, err = i.calcFile(name, algo)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
i.logFilePanic(name, err.Error())
|
||||
} else {
|
||||
i.new[name] = *info
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Index) showIgnoredOnly(ignore *Ignore) {
|
||||
for _, name := range i.files {
|
||||
if ignore.shouldIgnore(name) {
|
||||
i.logFile(STATUS_IGNORE, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Index) checkFix(force bool) {
|
||||
for name, b := range i.new {
|
||||
if a, ok := i.cur[name]; !ok {
|
||||
i.logFile(STATUS_NEW, name)
|
||||
i.setMod(true)
|
||||
continue
|
||||
} else {
|
||||
amod := int64(a.ModTime)
|
||||
bmod := int64(b.ModTime)
|
||||
if a.Hash != nil && b.Hash != nil && *a.Hash == *b.Hash {
|
||||
i.logFile(STATUS_OK, name)
|
||||
if amod != bmod {
|
||||
i.setMod(true)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if amod == bmod {
|
||||
i.logFile(STATUS_ERR_DMG, name)
|
||||
if !force {
|
||||
i.new[name] = a
|
||||
} else {
|
||||
i.setMod(true)
|
||||
}
|
||||
} else if amod < bmod {
|
||||
i.logFile(STATUS_UPDATE, name)
|
||||
i.setMod(true)
|
||||
} else if amod > bmod {
|
||||
i.logFile(STATUS_WARN_OLD, name)
|
||||
i.setMod(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Index) calcFile(name string, a string) (*IdxInfo, error) {
|
||||
path := filepath.Join(i.path, name)
|
||||
info, _ := os.Stat(path)
|
||||
mtime := int64(info.ModTime().UnixNano() / 1e6)
|
||||
h, err := Hashfile(path, a, i.context.perfMonBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i.context.perfMonFiles(1)
|
||||
return &IdxInfo{
|
||||
ModTime: mtime,
|
||||
Algo: &a,
|
||||
Hash: &h,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (i *Index) save() (bool, error) {
|
||||
if i.modified {
|
||||
if i.readonly {
|
||||
return false, errors.New("Error trying to save a readonly index.")
|
||||
}
|
||||
|
||||
text, err := json.Marshal(i.new)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
data := IndexFile{
|
||||
V: VERSION,
|
||||
IdxRaw: text,
|
||||
IdxHash: HashMd5(text),
|
||||
}
|
||||
|
||||
file, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = os.WriteFile(i.getIndexFilepath(), file, 0644)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
i.setMod(false)
|
||||
return true, nil
|
||||
} else {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Index) load() error {
|
||||
if _, err := os.Stat(i.getIndexFilepath()); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
i.setMod(false)
|
||||
file, err := os.ReadFile(i.getIndexFilepath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var data IndexFile
|
||||
err = json.Unmarshal(file, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if data.IdxRaw != nil {
|
||||
err = json.Unmarshal(data.IdxRaw, &i.cur)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := data.IdxRaw
|
||||
if data.IdxHash != HashMd5(text) {
|
||||
// old versions may have save the JSON encoded with extra spaces
|
||||
text, _ = json.Marshal(data.IdxRaw)
|
||||
} else {
|
||||
}
|
||||
if data.IdxHash != HashMd5(text) {
|
||||
i.setMod(true)
|
||||
i.logFile(STATUS_ERR_IDX, i.getIndexFilepath())
|
||||
}
|
||||
} else {
|
||||
var data1 IndexFile1
|
||||
json.Unmarshal(file, &data1)
|
||||
if data1.Data != nil {
|
||||
// convert from js to new format
|
||||
for name, item := range data1.Data {
|
||||
i.cur[name] = IdxInfo{
|
||||
ModTime: item.ModTime,
|
||||
Algo: &algoMd5,
|
||||
Hash: &item.Hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
37
check/status.go
Normal file
37
check/status.go
Normal file
@ -0,0 +1,37 @@
|
||||
package check
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
STATUS_PANIC Status = "EXC"
|
||||
STATUS_ERR_DMG Status = "DMG"
|
||||
STATUS_ERR_IDX Status = "EIX"
|
||||
STATUS_WARN_OLD Status = "old"
|
||||
STATUS_NEW Status = "new"
|
||||
STATUS_UPDATE Status = "upd"
|
||||
STATUS_OK Status = "ok "
|
||||
STATUS_IGNORE Status = "ign"
|
||||
STATUS_UPDATE_INDEX Status = "iup"
|
||||
)
|
||||
|
||||
func (s Status) String() string {
|
||||
return (string)(s)
|
||||
}
|
||||
|
||||
func (s Status) IsErrorOrWarning() bool {
|
||||
return s == STATUS_PANIC || s == STATUS_ERR_DMG || s == STATUS_ERR_IDX || s == STATUS_WARN_OLD
|
||||
}
|
||||
|
||||
func (s Status) IsVerbose() bool {
|
||||
return s == STATUS_OK || s == STATUS_IGNORE
|
||||
}
|
||||
|
||||
type LogEvent struct {
|
||||
Stat Status
|
||||
Message string
|
||||
}
|
||||
|
||||
type PerfEvent struct {
|
||||
NumFiles int64
|
||||
NumBytes int64
|
||||
}
|
37
check/worker.go
Normal file
37
check/worker.go
Normal file
@ -0,0 +1,37 @@
|
||||
package check
|
||||
|
||||
type WorkItem struct {
|
||||
path string
|
||||
filesToIndex []string
|
||||
ignore *Ignore
|
||||
}
|
||||
|
||||
func (context *Context) RunWorker(id int) {
|
||||
for {
|
||||
item := <-context.WorkQueue
|
||||
if item == nil {
|
||||
break
|
||||
}
|
||||
|
||||
index := NewIndex(context, item.path, item.filesToIndex, !context.Update)
|
||||
err := index.load()
|
||||
if err != nil {
|
||||
context.log(STATUS_PANIC, index.getIndexFilepath()+": "+err.Error())
|
||||
}
|
||||
|
||||
if context.ShowIgnoredOnly {
|
||||
index.showIgnoredOnly(item.ignore)
|
||||
} else {
|
||||
index.calcHashes(item.ignore)
|
||||
index.checkFix(context.Force)
|
||||
|
||||
if context.Update {
|
||||
if changed, err := index.save(); err != nil {
|
||||
context.logErr(item.path, err)
|
||||
} else if changed {
|
||||
context.log(STATUS_UPDATE_INDEX, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
24
cmd/chkbit/help.go
Normal file
24
cmd/chkbit/help.go
Normal file
@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
var headerHelp = `Checks the data integrity of your files.
|
||||
For help tips run "chkbit -H" or go to
|
||||
https://github.com/laktak/chkbit
|
||||
`
|
||||
|
||||
var helpTips = `
|
||||
.chkbitignore rules:
|
||||
each line should contain exactly one name
|
||||
you may use Unix shell-style wildcards (see README)
|
||||
lines starting with '#' are skipped
|
||||
lines starting with '/' are only applied to the current directory
|
||||
|
||||
Status codes:
|
||||
DMG: error, data damage detected
|
||||
EIX: error, index damaged
|
||||
old: warning, file replaced by an older version
|
||||
new: new file
|
||||
upd: file updated
|
||||
ok : check ok
|
||||
ign: ignored (see .chkbitignore)
|
||||
EXC: exception/panic
|
||||
`
|
335
cmd/chkbit/main.go
Normal file
335
cmd/chkbit/main.go
Normal file
@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/laktak/chkbit/check"
|
||||
"github.com/laktak/chkbit/term"
|
||||
"github.com/laktak/chkbit/util"
|
||||
)
|
||||
|
||||
type Progress int
|
||||
|
||||
const (
|
||||
Quiet Progress = iota
|
||||
Summary
|
||||
Plain
|
||||
Fancy
|
||||
)
|
||||
|
||||
const (
|
||||
updateInterval = time.Millisecond * 300
|
||||
sizeMB int64 = 1024 * 1024
|
||||
)
|
||||
|
||||
var appVersion = "vdev"
|
||||
var (
|
||||
termBG = term.Bg8(240)
|
||||
termSep = "|"
|
||||
termSepFG = term.Fg8(235)
|
||||
termFG1 = term.Fg8(255)
|
||||
termFG2 = term.Fg8(228)
|
||||
termFG3 = term.Fg8(202)
|
||||
termOKFG = term.Fg4(2)
|
||||
termAlertFG = term.Fg4(1)
|
||||
)
|
||||
|
||||
var cli struct {
|
||||
Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"`
|
||||
Tips bool `short:"H" help:"Show tips."`
|
||||
Update bool `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"`
|
||||
ShowIgnoredOnly bool `help:"only show ignored files"`
|
||||
Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
|
||||
Force bool `short:"f" help:"force update of damaged items"`
|
||||
SkipSymlinks bool `short:"s" help:"do not follow symlinks"`
|
||||
LogFile string `short:"l" help:"write to a logfile if specified"`
|
||||
LogVerbose bool `help:"verbose logging"`
|
||||
IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
|
||||
IgnoreName string `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"`
|
||||
Workers int `short:"w" default:"5" help:"number of workers to use (default: 5)"`
|
||||
Plain bool `help:"show plain status instead of being fancy"`
|
||||
Quiet bool `short:"q" help:"quiet, don't show progress/information"`
|
||||
Verbose bool `short:"v" help:"verbose output"`
|
||||
Version bool `short:"V" help:"show version information"`
|
||||
}
|
||||
|
||||
type Main struct {
|
||||
dmgList []string
|
||||
errList []string
|
||||
numIdxUpd int
|
||||
numNew int
|
||||
numUpd int
|
||||
verbose bool
|
||||
logger *log.Logger
|
||||
logVerbose bool
|
||||
progress Progress
|
||||
total int
|
||||
termWidth int
|
||||
fps *RateCalc
|
||||
bps *RateCalc
|
||||
}
|
||||
|
||||
func (m *Main) log(text string) {
|
||||
m.logger.Println(time.Now().UTC().Format("2006-01-02 15:04:05"), text)
|
||||
}
|
||||
|
||||
func (m *Main) logStatus(stat check.Status, path string) {
|
||||
if stat == check.STATUS_UPDATE_INDEX {
|
||||
m.numIdxUpd++
|
||||
} else {
|
||||
if stat == check.STATUS_ERR_DMG {
|
||||
m.total++
|
||||
m.dmgList = append(m.dmgList, path)
|
||||
} else if stat == check.STATUS_PANIC {
|
||||
m.errList = append(m.errList, path)
|
||||
} else if stat == check.STATUS_OK || stat == check.STATUS_UPDATE || stat == check.STATUS_NEW {
|
||||
m.total++
|
||||
if stat == check.STATUS_UPDATE {
|
||||
m.numUpd++
|
||||
} else if stat == check.STATUS_NEW {
|
||||
m.numNew++
|
||||
}
|
||||
}
|
||||
|
||||
if m.logVerbose || stat != check.STATUS_OK && stat != check.STATUS_IGNORE {
|
||||
m.log(stat.String() + " " + path)
|
||||
}
|
||||
|
||||
if m.verbose || !stat.IsVerbose() {
|
||||
col := ""
|
||||
if stat.IsErrorOrWarning() {
|
||||
col = termAlertFG
|
||||
}
|
||||
term.Printline(col, stat.String(), " ", path, term.Reset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Main) showStatus(context *check.Context) {
|
||||
last := time.Now().Add(-updateInterval)
|
||||
stat := ""
|
||||
for {
|
||||
select {
|
||||
case item := <-context.LogQueue:
|
||||
if item == nil {
|
||||
if m.progress == Fancy {
|
||||
term.Printline("")
|
||||
}
|
||||
return
|
||||
}
|
||||
m.logStatus(item.Stat, item.Message)
|
||||
if m.progress == Fancy {
|
||||
term.Write(termBG, termFG1, stat, term.ClearLine(0), term.Reset, "\r")
|
||||
} else {
|
||||
fmt.Print(m.total, "\r")
|
||||
}
|
||||
case perf := <-context.PerfQueue:
|
||||
now := time.Now()
|
||||
m.fps.Push(now, perf.NumFiles)
|
||||
m.bps.Push(now, perf.NumBytes)
|
||||
if last.Add(updateInterval).Before(now) {
|
||||
last = now
|
||||
if m.progress == Fancy {
|
||||
statF := fmt.Sprintf("%d files/s", m.fps.Last())
|
||||
statB := fmt.Sprintf("%d MB/s", m.bps.Last()/sizeMB)
|
||||
stat = "RW"
|
||||
if !context.Update {
|
||||
stat = "RO"
|
||||
}
|
||||
stat = fmt.Sprintf("[%s:%d] %5d files $ %s %-13s $ %s %-13s",
|
||||
stat, context.NumWorkers, m.total,
|
||||
util.Sparkline(m.fps.Stats), statF,
|
||||
util.Sparkline(m.bps.Stats), statB)
|
||||
stat = util.LeftTruncate(stat, m.termWidth-1)
|
||||
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG2, 1)
|
||||
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG3, 1)
|
||||
term.Write(termBG, termFG1, stat, term.ClearLine(0), term.Reset, "\r")
|
||||
} else if m.progress == Plain {
|
||||
fmt.Print(m.total, "\r")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Main) process() *check.Context {
|
||||
if cli.Update && cli.ShowIgnoredOnly {
|
||||
fmt.Println("Error: use either --update or --show-ignored-only!")
|
||||
return nil
|
||||
}
|
||||
|
||||
context, err := check.NewContext(cli.Workers, cli.Force, cli.Update, cli.ShowIgnoredOnly, cli.Algo, cli.SkipSymlinks, cli.IndexName, cli.IgnoreName)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
m.showStatus(context)
|
||||
}()
|
||||
context.Start(cli.Paths)
|
||||
wg.Wait()
|
||||
|
||||
return context
|
||||
}
|
||||
|
||||
func (m *Main) printResult(context *check.Context) {
|
||||
cprint := func(col, text string) {
|
||||
if m.progress != Quiet {
|
||||
if m.progress == Fancy {
|
||||
term.Printline(col, text, term.Reset)
|
||||
} else {
|
||||
fmt.Println(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eprint := func(col, text string) {
|
||||
if m.progress == Fancy {
|
||||
term.Write(col)
|
||||
fmt.Fprintln(os.Stderr, text)
|
||||
term.Write(term.Reset)
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, text)
|
||||
}
|
||||
}
|
||||
|
||||
if m.progress != Quiet {
|
||||
mode := ""
|
||||
if !context.Update {
|
||||
mode = " in readonly mode"
|
||||
}
|
||||
status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(m.total, "file"), mode)
|
||||
cprint(termOKFG, status)
|
||||
m.log(status)
|
||||
|
||||
if m.progress == Fancy && m.total > 0 {
|
||||
elapsed := time.Since(m.fps.Start)
|
||||
elapsedS := elapsed.Seconds()
|
||||
fmt.Println("-", elapsed.Truncate(time.Second), "elapsed")
|
||||
fmt.Printf("- %.2f files/second\n", (float64(m.fps.Total)+float64(m.fps.Current))/elapsedS)
|
||||
fmt.Printf("- %.2f MB/second\n", (float64(m.bps.Total)+float64(m.bps.Current))/float64(sizeMB)/elapsedS)
|
||||
}
|
||||
|
||||
if context.Update {
|
||||
if m.numIdxUpd > 0 {
|
||||
cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated",
|
||||
util.LangNum1Choice(m.numIdxUpd, "directory was", "directories were"),
|
||||
util.LangNum1Choice(m.numNew, "file hash was", "file hashes were"),
|
||||
util.LangNum1Choice(m.numUpd, "file hash was", "file hashes were")))
|
||||
}
|
||||
} else if m.numNew+m.numUpd > 0 {
|
||||
cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added and\n- %s would have been updated.",
|
||||
util.LangNum1MutateSuffix(m.numNew, "file"),
|
||||
util.LangNum1MutateSuffix(m.numUpd, "file")))
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.dmgList) > 0 {
|
||||
eprint(termAlertFG, "chkbit detected damage in these files:")
|
||||
for _, err := range m.dmgList {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
n := len(m.dmgList)
|
||||
status := fmt.Sprintf("error: detected %s with damage!", util.LangNum1MutateSuffix(n, "file"))
|
||||
m.log(status)
|
||||
eprint(termAlertFG, status)
|
||||
}
|
||||
|
||||
if len(m.errList) > 0 {
|
||||
status := "chkbit ran into errors"
|
||||
m.log(status + "!")
|
||||
eprint(termAlertFG, status+":")
|
||||
for _, err := range m.errList {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.dmgList) > 0 || len(m.errList) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Main) run() {
|
||||
|
||||
if len(os.Args) < 2 {
|
||||
os.Args = append(os.Args, "--help")
|
||||
}
|
||||
|
||||
kong.Parse(&cli,
|
||||
kong.Name("chkbit"),
|
||||
kong.Description(""),
|
||||
kong.UsageOnError(),
|
||||
)
|
||||
|
||||
if cli.Tips {
|
||||
fmt.Println(helpTips)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if cli.Version {
|
||||
fmt.Println("github.com/laktak/chkbit")
|
||||
fmt.Println(appVersion)
|
||||
return
|
||||
}
|
||||
|
||||
m.verbose = cli.Verbose || cli.ShowIgnoredOnly
|
||||
if cli.LogFile != "" {
|
||||
m.logVerbose = cli.LogVerbose
|
||||
f, err := os.OpenFile(cli.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
m.logger = log.New(f, "", 0)
|
||||
}
|
||||
|
||||
if cli.Quiet {
|
||||
m.progress = Quiet
|
||||
} else if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
|
||||
m.progress = Summary
|
||||
} else if cli.Plain {
|
||||
m.progress = Plain
|
||||
} else {
|
||||
m.progress = Fancy
|
||||
}
|
||||
|
||||
if len(cli.Paths) > 0 {
|
||||
m.log("chkbit " + strings.Join(cli.Paths, ", "))
|
||||
context := m.process()
|
||||
if context != nil && !context.ShowIgnoredOnly {
|
||||
m.printResult(context)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("specify a path to check, see -h")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
fmt.Println(r)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
termWidth := term.GetWidth()
|
||||
m := &Main{
|
||||
logger: log.New(io.Discard, "", 0),
|
||||
termWidth: termWidth,
|
||||
fps: NewRateCalc(time.Second, (termWidth-70)/2),
|
||||
bps: NewRateCalc(time.Second, (termWidth-70)/2),
|
||||
}
|
||||
m.run()
|
||||
}
|
52
cmd/chkbit/rate_calc.go
Normal file
52
cmd/chkbit/rate_calc.go
Normal file
@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type RateCalc struct {
|
||||
Interval time.Duration
|
||||
MaxStat int
|
||||
Start time.Time
|
||||
Updated time.Time
|
||||
Total int64
|
||||
Current int64
|
||||
Stats []int64
|
||||
}
|
||||
|
||||
func NewRateCalc(interval time.Duration, maxStat int) *RateCalc {
|
||||
if maxStat < 10 {
|
||||
maxStat = 10
|
||||
}
|
||||
rc := &RateCalc{
|
||||
Interval: interval,
|
||||
MaxStat: maxStat,
|
||||
}
|
||||
rc.Reset()
|
||||
return rc
|
||||
}
|
||||
|
||||
func (rc *RateCalc) Reset() {
|
||||
rc.Start = time.Now()
|
||||
rc.Updated = rc.Start
|
||||
rc.Total = 0
|
||||
rc.Current = 0
|
||||
rc.Stats = make([]int64, rc.MaxStat)
|
||||
}
|
||||
|
||||
func (rc *RateCalc) Last() int64 {
|
||||
return rc.Stats[len(rc.Stats)-1]
|
||||
}
|
||||
|
||||
func (rc *RateCalc) Push(ts time.Time, value int64) {
|
||||
for rc.Updated.Add(rc.Interval).Before(ts) {
|
||||
rc.Stats = append(rc.Stats, rc.Current)
|
||||
if len(rc.Stats) > rc.MaxStat {
|
||||
rc.Stats = rc.Stats[len(rc.Stats)-rc.MaxStat:]
|
||||
}
|
||||
rc.Total += rc.Current
|
||||
rc.Current = 0
|
||||
rc.Updated = rc.Updated.Add(rc.Interval)
|
||||
}
|
||||
rc.Current += value
|
||||
}
|
14
go.mod
Normal file
14
go.mod
Normal file
@ -0,0 +1,14 @@
|
||||
module github.com/laktak/chkbit
|
||||
|
||||
go 1.22.3
|
||||
|
||||
require (
|
||||
github.com/alecthomas/kong v0.9.0
|
||||
golang.org/x/sys v0.23.0
|
||||
lukechampine.com/blake3 v1.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
golang.org/x/term v0.23.0 // indirect
|
||||
)
|
18
go.sum
Normal file
18
go.sum
Normal file
@ -0,0 +1,18 @@
|
||||
github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
|
||||
github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA=
|
||||
github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
|
||||
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
|
||||
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
|
||||
lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE=
|
||||
lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
|
8
scripts/build
Executable file
8
scripts/build
Executable file
@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -eE -o pipefail
|
||||
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
cd $script_dir/..
|
||||
|
||||
version=$(git describe --tags --always)
|
||||
go build -ldflags="-X main.appVersion=$version" ./cmd/chkbit
|
13
scripts/chkfmt
Executable file
13
scripts/chkfmt
Executable file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -eE -o pipefail
|
||||
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
cd $script_dir/..
|
||||
|
||||
res="$(gofmt -l . 2>&1)"
|
||||
|
||||
if [ -n "$res" ]; then
|
||||
echo "gofmt check failed:"
|
||||
echo "${res}"
|
||||
exit 1
|
||||
fi
|
7
scripts/lint
Executable file
7
scripts/lint
Executable file
@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -eE -o pipefail
|
||||
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
cd $script_dir/..
|
||||
|
||||
go vet -structtag=false -composites=false ./...
|
13
scripts/run_test_prep
Executable file
13
scripts/run_test_prep
Executable file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
export TZ='UTC'
|
||||
root="/tmp/chkbit"
|
||||
|
||||
go run scripts/run_test_prep.go
|
||||
|
||||
cd $root/root
|
||||
mv $root/root/people $root/people
|
||||
ln -s ../people people
|
||||
ln -s ../../people/face/office-door.pdf day/friend/office-door.pdf
|
||||
|
||||
find -L | wc -l
|
89
scripts/run_test_prep.go
Normal file
89
scripts/run_test_prep.go
Normal file
@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
startList = []string{"time", "year", "people", "way", "day", "thing"}
|
||||
wordList = []string{"life", "world", "school", "state", "family", "student", "group", "country", "problem", "hand", "part", "place", "case", "week", "company", "system", "program", "work", "government", "number", "night", "point", "home", "water", "room", "mother", "area", "money", "story", "fact", "month", "lot", "right", "study", "book", "eye", "job", "word", "business", "issue", "side", "kind", "head", "house", "service", "friend", "father", "power", "hour", "game", "line", "end", "member", "law", "car", "city", "community", "name", "president", "team", "minute", "idea", "kid", "body", "information", "back", "face", "others", "level", "office", "door", "health", "person", "art", "war", "history", "party", "result", "change", "morning", "reason", "research", "moment", "air", "teacher", "force", "education"}
|
||||
extList = []string{"txt", "md", "pdf", "jpg", "jpeg", "png", "mp4", "mp3", "csv"}
|
||||
startDate = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
endDate = time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC)
|
||||
dateList = []time.Time{}
|
||||
wordIdx = 0
|
||||
extIdx = 0
|
||||
dateIdx = 0
|
||||
)
|
||||
|
||||
func nextWord() string {
|
||||
word := wordList[wordIdx%len(wordList)]
|
||||
wordIdx++
|
||||
return word
|
||||
}
|
||||
|
||||
func nextExt() string {
|
||||
ext := extList[extIdx%len(extList)]
|
||||
extIdx++
|
||||
return ext
|
||||
}
|
||||
|
||||
func setDate(filename string, r int) {
|
||||
date := dateList[dateIdx%len(dateList)]
|
||||
m := 17 * dateIdx / len(dateList)
|
||||
date = date.Add(time.Duration(m) * time.Hour)
|
||||
dateIdx++
|
||||
os.Chtimes(filename, date, date)
|
||||
}
|
||||
|
||||
func genFile(dir string, a int) {
|
||||
os.MkdirAll(dir, 0755)
|
||||
for i := 1; i <= 5; i++ {
|
||||
size := a*i*wordIdx*100 + extIdx
|
||||
file := nextWord() + "-" + nextWord()
|
||||
|
||||
if i%3 == 0 {
|
||||
file += "-" + nextWord()
|
||||
}
|
||||
|
||||
file += "." + nextExt()
|
||||
path := filepath.Join(dir, file)
|
||||
os.WriteFile(path, make([]byte, size), 0644)
|
||||
setDate(path, size*size)
|
||||
}
|
||||
}
|
||||
|
||||
func genDir(root string) {
|
||||
for _, start := range startList {
|
||||
|
||||
for i := 1; i <= 5; i++ {
|
||||
dir := filepath.Join(root, start, nextWord())
|
||||
genFile(dir, 1)
|
||||
|
||||
if wordIdx%3 == 0 {
|
||||
dir = filepath.Join(dir, nextWord())
|
||||
genFile(dir, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "/tmp/chkbit"
|
||||
|
||||
var c int64 = 50
|
||||
interval := (int64)(endDate.Sub(startDate).Seconds()) / c
|
||||
for i := range make([]int64, c) {
|
||||
dateList = append(dateList, startDate.Add(time.Duration(interval*(int64)(i))*time.Second))
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(root); err == nil {
|
||||
genDir(filepath.Join(root, "root"))
|
||||
fmt.Println("Ready.")
|
||||
} else {
|
||||
fmt.Println("Failed to clean")
|
||||
}
|
||||
}
|
19
scripts/run_tests
Executable file
19
scripts/run_tests
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
export TZ='UTC'
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
base_dir=$(dirname "$script_dir")
|
||||
#dir=$(realpath "$script_dir/../testdata/run_test")
|
||||
root="/tmp/chkbit/root"
|
||||
|
||||
if [[ ! -d $root ]]; then
|
||||
echo "must run run_test_prep first"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# setup
|
||||
|
||||
$script_dir/build
|
||||
|
||||
"$base_dir/chkbit" -u /tmp/chkbit
|
7
scripts/tests
Executable file
7
scripts/tests
Executable file
@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
cd $script_dir/..
|
||||
|
||||
go test -v ./util
|
55
scripts/xbuild
Executable file
55
scripts/xbuild
Executable file
@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
set -eE -o pipefail
|
||||
|
||||
script_dir=$(dirname "$(realpath "$0")")
|
||||
cd $script_dir/..
|
||||
|
||||
if [ -z "$version" ]; then
|
||||
version=$(git rev-parse HEAD)
|
||||
fi
|
||||
|
||||
echo "building version $version"
|
||||
|
||||
mkdir -p dist
|
||||
rm -f dist/*
|
||||
|
||||
build() {
|
||||
echo "- $1-$2"
|
||||
rm -f dist/chkbit
|
||||
CGO_ENABLED=0 GOOS="$1" GOARCH="$2" go build -o dist -ldflags="-X main.appVersion=$version" ./cmd/chkbit
|
||||
|
||||
pushd dist
|
||||
|
||||
case "$1" in
|
||||
windows)
|
||||
outfile="chkbit-$1-$2.zip"
|
||||
zip "$outfile" chkbit.exe --move
|
||||
;;
|
||||
*)
|
||||
outfile="chkbit-$1-$2.tar.gz"
|
||||
tar -czf "$outfile" chkbit --remove-files
|
||||
;;
|
||||
esac
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
if [[ -z $2 ]]; then
|
||||
build android arm64
|
||||
build darwin amd64
|
||||
build darwin arm64
|
||||
build freebsd amd64
|
||||
build freebsd arm64
|
||||
build freebsd riscv64
|
||||
build linux amd64
|
||||
build linux arm64
|
||||
build linux riscv64
|
||||
build netbsd amd64
|
||||
build netbsd arm64
|
||||
build openbsd amd64
|
||||
build openbsd arm64
|
||||
build windows amd64
|
||||
build windows arm64
|
||||
else
|
||||
build $1 $2
|
||||
fi
|
82
term/term.go
Normal file
82
term/term.go
Normal file
@ -0,0 +1,82 @@
|
||||
package term
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
var (
|
||||
isTerm = false
|
||||
noColor = false
|
||||
stdoutFd = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
stdoutFd = int(os.Stdout.Fd())
|
||||
isTerm = term.IsTerminal(stdoutFd)
|
||||
if isTerm {
|
||||
noColor = os.Getenv("NO_COLOR") != ""
|
||||
} else {
|
||||
noColor = true
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
Reset = "\033[0m"
|
||||
Bold = "\033[01m"
|
||||
Disable = "\033[02m"
|
||||
Underline = "\033[04m"
|
||||
Reverse = "\033[07m"
|
||||
Strikethrough = "\033[09m"
|
||||
Invisible = "\033[08m"
|
||||
)
|
||||
|
||||
func Write(text ...interface{}) {
|
||||
fmt.Print(text...)
|
||||
}
|
||||
|
||||
func Printline(text ...interface{}) {
|
||||
fmt.Print(text...)
|
||||
fmt.Println(ClearLine(0))
|
||||
}
|
||||
|
||||
func Fg4(col int) string {
|
||||
if noColor {
|
||||
return ""
|
||||
}
|
||||
if col < 8 {
|
||||
return fmt.Sprintf("\033[%dm", 30+col)
|
||||
}
|
||||
return fmt.Sprintf("\033[%dm", 90-8+col)
|
||||
}
|
||||
|
||||
func Fg8(col int) string {
|
||||
if noColor {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("\033[38;5;%dm", col)
|
||||
}
|
||||
|
||||
func Bg8(col int) string {
|
||||
if noColor {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("\033[48;5;%dm", col)
|
||||
}
|
||||
|
||||
func ClearLine(opt int) string {
|
||||
// 0=to end, 1=from start, 2=all
|
||||
return fmt.Sprintf("\033[%dK", opt)
|
||||
}
|
||||
|
||||
func GetWidth() int {
|
||||
if isTerm {
|
||||
width, _, err := term.GetSize(stdoutFd)
|
||||
if err == nil {
|
||||
return width
|
||||
}
|
||||
}
|
||||
return 80
|
||||
}
|
21
term/term_windows.go
Normal file
21
term/term_windows.go
Normal file
@ -0,0 +1,21 @@
|
||||
package term
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// from https://github.com/fatih/color
|
||||
|
||||
func init() {
|
||||
// Opt-in for ansi color support for current process.
|
||||
// https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences
|
||||
var outMode uint32
|
||||
out := windows.Handle(os.Stdout.Fd())
|
||||
if err := windows.GetConsoleMode(out, &outMode); err != nil {
|
||||
return
|
||||
}
|
||||
outMode |= windows.ENABLE_PROCESSED_OUTPUT | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING
|
||||
_ = windows.SetConsoleMode(out, outMode)
|
||||
}
|
35
util/fm.go
Normal file
35
util/fm.go
Normal file
@ -0,0 +1,35 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
func Minimum(series []int64) int64 {
|
||||
var min int64 = math.MaxInt64
|
||||
for _, value := range series {
|
||||
if value < min {
|
||||
min = value
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func Maximum(series []int64) int64 {
|
||||
var max int64 = math.MinInt64
|
||||
for _, value := range series {
|
||||
if value > max {
|
||||
max = value
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func Clamp(min int64, max int64, n int64) int64 {
|
||||
if n < min {
|
||||
return min
|
||||
}
|
||||
if n > max {
|
||||
return max
|
||||
}
|
||||
return n
|
||||
}
|
19
util/lang.go
Normal file
19
util/lang.go
Normal file
@ -0,0 +1,19 @@
|
||||
package util
|
||||
|
||||
import "fmt"
|
||||
|
||||
func LangNum1MutateSuffix(num int, u string) string {
|
||||
s := ""
|
||||
if num != 1 {
|
||||
s = "s"
|
||||
}
|
||||
return fmt.Sprintf("%d %s%s", num, u, s)
|
||||
}
|
||||
|
||||
func LangNum1Choice(num int, u1, u2 string) string {
|
||||
u := u1
|
||||
if num != 1 {
|
||||
u = u2
|
||||
}
|
||||
return fmt.Sprintf("%d %s", num, u)
|
||||
}
|
32
util/sparkline.go
Normal file
32
util/sparkline.go
Normal file
@ -0,0 +1,32 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
var sparkChars = []rune{'▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'}
|
||||
|
||||
func Sparkline(series []int64) string {
|
||||
out := make([]rune, len(series))
|
||||
min := Minimum(series)
|
||||
max := Maximum(series)
|
||||
dataRange := max - min
|
||||
if dataRange == 0 {
|
||||
for i := range series {
|
||||
out[i] = sparkChars[0]
|
||||
}
|
||||
} else {
|
||||
step := float64(len(sparkChars)-1) / float64(dataRange)
|
||||
for i, n := range series {
|
||||
idx := int(math.Round(float64(Clamp(min, max, n)-min) * step))
|
||||
if idx < 0 {
|
||||
out[i] = ' '
|
||||
} else if idx > len(sparkChars) {
|
||||
out[i] = sparkChars[len(sparkChars)-1]
|
||||
} else {
|
||||
out[i] = sparkChars[idx]
|
||||
}
|
||||
}
|
||||
}
|
||||
return string(out)
|
||||
}
|
13
util/sparkline_test.go
Normal file
13
util/sparkline_test.go
Normal file
@ -0,0 +1,13 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpark(t *testing.T) {
|
||||
expected := "▁▁▂▄▅▇██▆▄▂"
|
||||
actual := Sparkline([]int64{5, 12, 35, 73, 80, 125, 150, 142, 118, 61, 19})
|
||||
if expected != actual {
|
||||
t.Error("expected:", expected, "actual:", actual)
|
||||
}
|
||||
}
|
11
util/strings.go
Normal file
11
util/strings.go
Normal file
@ -0,0 +1,11 @@
|
||||
package util
|
||||
|
||||
func LeftTruncate(s string, nMax int) string {
|
||||
for i := range s {
|
||||
nMax--
|
||||
if nMax < 0 {
|
||||
return s[:i]
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
13
util/strings_test.go
Normal file
13
util/strings_test.go
Normal file
@ -0,0 +1,13 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTrunc(t *testing.T) {
|
||||
expected := "ab©def"
|
||||
actual := LeftTruncate(expected+"ghijk", 6)
|
||||
if expected != actual {
|
||||
t.Error("expected:", expected, "actual:", actual)
|
||||
}
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user