Compare commits
No commits in common. "main" and "1.0.0" have entirely different histories.
30
README.rst
30
README.rst
@ -49,38 +49,12 @@ Tests
|
||||
-----
|
||||
|
||||
There's a simple but comprehensive test scenario using
|
||||
`pytest <https://pypi.org/p/pytest>`_ and
|
||||
`pytest-order <https://pypi.org/p/pytest-order>`_.
|
||||
|
||||
Install::
|
||||
|
||||
$ python3 -m venv .venv
|
||||
$ . .venv/bin/activate
|
||||
(.venv)$ pip install -e .[test]
|
||||
|
||||
Run::
|
||||
|
||||
(.venv)$ pytest -x
|
||||
==================== test session starts ====================
|
||||
platform darwin -- Python 3.10.12, pytest-7.4.0, pluggy-1.2.0
|
||||
rootdir: /Users/ambv/Documents/Python/bitrot
|
||||
plugins: order-1.1.0
|
||||
collected 12 items
|
||||
|
||||
tests/test_bitrot.py ............ [100%]
|
||||
|
||||
==================== 12 passed in 15.05s ====================
|
||||
`BATS <https://github.com/sstephenson/bats>`. Run the
|
||||
file in the `tests` directory to run it.
|
||||
|
||||
Change Log
|
||||
----------
|
||||
|
||||
1.0.1
|
||||
~~~~~
|
||||
|
||||
* officially remove Python 2 support that was broken since 1.0.0
|
||||
anyway; now the package works with Python 3.8+ because of a few
|
||||
features
|
||||
|
||||
1.0.0
|
||||
~~~~~
|
||||
|
||||
|
||||
36
bin/bitrot
Normal file
36
bin/bitrot
Normal file
@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (C) 2013 by Łukasz Langa
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from multiprocessing import freeze_support
|
||||
|
||||
from bitrot import run_from_command_line
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
freeze_support()
|
||||
run_from_command_line()
|
||||
@ -1,34 +0,0 @@
|
||||
[build-system]
|
||||
requires = ["setuptools", "setuptools-scm[toml]"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "bitrot"
|
||||
authors = [
|
||||
{name = "Łukasz Langa", email = "lukasz@langa.pl"},
|
||||
]
|
||||
description = "Detects bit rotten files on the hard drive to save your precious photo and music collection from slow decay."
|
||||
readme = "README.rst"
|
||||
requires-python = ">=3.8"
|
||||
keywords = ["file", "checksum", "database"]
|
||||
license = {text = "MIT"}
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Natural Language :: English",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Topic :: System :: Filesystems",
|
||||
"Topic :: System :: Monitoring",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
|
||||
]
|
||||
dependencies = []
|
||||
dynamic = ["version"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest", "pytest-order"]
|
||||
|
||||
[project.scripts]
|
||||
bitrot = "bitrot:run_from_command_line"
|
||||
|
||||
[tool.setuptools_scm]
|
||||
tag_regex = "^(?P<version>v\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$"
|
||||
76
setup.py
Normal file
76
setup.py
Normal file
@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (C) 2013 by Łukasz Langa
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import codecs
|
||||
import os
|
||||
import sys
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
current_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
ld_file = codecs.open(os.path.join(current_dir, 'README.rst'), encoding='utf8')
|
||||
try:
|
||||
long_description = ld_file.read()
|
||||
finally:
|
||||
ld_file.close()
|
||||
# We let it die a horrible tracebacking death if reading the file fails.
|
||||
# We couldn't sensibly recover anyway: we need the long description.
|
||||
|
||||
sys.path.insert(0, current_dir + os.sep + 'src')
|
||||
from bitrot import VERSION
|
||||
release = ".".join(str(num) for num in VERSION)
|
||||
|
||||
setup(
|
||||
name = 'bitrot',
|
||||
version = release,
|
||||
author = u'Łukasz Langa',
|
||||
author_email = 'lukasz@langa.pl',
|
||||
description = ("Detects bit rotten files on the hard drive to save your "
|
||||
"precious photo and music collection from slow decay."),
|
||||
long_description = long_description,
|
||||
url = 'https://github.com/ambv/bitrot/',
|
||||
keywords = 'file checksum database',
|
||||
platforms = ['any'],
|
||||
license = 'MIT',
|
||||
package_dir = {'': 'src'},
|
||||
packages = find_packages('src'),
|
||||
py_modules = ['bitrot'],
|
||||
scripts = ['bin/bitrot'],
|
||||
include_package_data = True,
|
||||
zip_safe = False, # if only because of the readme file
|
||||
install_requires = [
|
||||
'futures; python_version == "2.7"'
|
||||
],
|
||||
classifiers = [
|
||||
'Development Status :: 4 - Beta',
|
||||
'License :: OSI Approved :: MIT License',
|
||||
'Natural Language :: English',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python',
|
||||
'Topic :: System :: Filesystems',
|
||||
'Topic :: System :: Monitoring',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
]
|
||||
)
|
||||
@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (C) 2013 by Łukasz Langa
|
||||
|
||||
@ -20,7 +21,10 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
@ -36,19 +40,19 @@ import tempfile
|
||||
import time
|
||||
import unicodedata
|
||||
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from multiprocessing import freeze_support
|
||||
from importlib.metadata import version, PackageNotFoundError
|
||||
from concurrent.futures import ProcessPoolExecutor, wait, as_completed
|
||||
|
||||
|
||||
DEFAULT_CHUNK_SIZE = 16384 # block size in HFS+; 4X the block size in ext4
|
||||
DOT_THRESHOLD = 200
|
||||
VERSION = (1, 0, 0)
|
||||
IGNORED_FILE_SYSTEM_ERRORS = {errno.ENOENT, errno.EACCES}
|
||||
FSENCODING = sys.getfilesystemencoding()
|
||||
try:
|
||||
VERSION = version("bitrot")
|
||||
except PackageNotFoundError:
|
||||
VERSION = "1.0.1"
|
||||
|
||||
|
||||
if sys.version[0] == '2':
|
||||
str = type(u'text')
|
||||
# use `bytes` for bytestrings
|
||||
|
||||
|
||||
def normalize_path(path):
|
||||
@ -163,6 +167,8 @@ def compute_one(path, chunk_size):
|
||||
|
||||
raise # Not expected? https://github.com/ambv/bitrot/issues/
|
||||
|
||||
new_mtime = int(st.st_mtime)
|
||||
|
||||
try:
|
||||
new_sha1 = sha1(path, chunk_size)
|
||||
except (IOError, OSError) as e:
|
||||
@ -516,8 +522,6 @@ def update_sha512_integrity(verbosity=1):
|
||||
def run_from_command_line():
|
||||
global FSENCODING
|
||||
|
||||
freeze_support()
|
||||
|
||||
parser = argparse.ArgumentParser(prog='bitrot')
|
||||
parser.add_argument(
|
||||
'-l', '--follow-links', action='store_true',
|
||||
@ -543,7 +547,7 @@ def run_from_command_line():
|
||||
help='just test against an existing database, don\'t update anything')
|
||||
parser.add_argument(
|
||||
'--version', action='version',
|
||||
version=f"%(prog)s {VERSION}")
|
||||
version='%(prog)s {}.{}.{}'.format(*VERSION))
|
||||
parser.add_argument(
|
||||
'--commit-interval', type=float, default=300,
|
||||
help='min time in seconds between commits '
|
||||
|
||||
212
tests/test-bitrot.bats
Executable file
212
tests/test-bitrot.bats
Executable file
@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
LC_ALL=en_US.UTF-8
|
||||
LANG=en_US.UTF-8
|
||||
|
||||
cmd='python -m bitrot'
|
||||
test_dir=/tmp/bitrot_dir-$USER
|
||||
mkdir -p $test_dir
|
||||
cd $test_dir || exit
|
||||
|
||||
@test "bitrot command exists" {
|
||||
run $cmd --help
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "bitrot detects new files in a tree dir" {
|
||||
mkdir -p nonemptydirs/dir2/
|
||||
touch nonemptydirs/dir2/new-file-{a,b}.txt
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
# [[ ${lines[0]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[1]} = "2 entries in the database. 2 entries new:" ]]
|
||||
[[ ${lines[2]} = " ./nonemptydirs/dir2/new-file-a.txt" ]]
|
||||
[[ ${lines[3]} = " ./nonemptydirs/dir2/new-file-b.txt" ]]
|
||||
[[ ${lines[4]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
@test "bitrot detects modified files in a tree dir" {
|
||||
sleep 2
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-a.txt
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[0]} = "Checking bitrot.db integrity... ok." ]]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "2 entries in the database. 1 entries updated:" ]]
|
||||
[[ ${lines[3]} = " ./nonemptydirs/dir2/new-file-a.txt" ]]
|
||||
[[ ${lines[4]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
@test "bitrot detects renamed files in a tree dir" {
|
||||
sleep 1
|
||||
mv nonemptydirs/dir2/new-file-a.txt nonemptydirs/dir2/new-file-a.txt2
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[0]} = "Checking bitrot.db integrity... ok." ]]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "2 entries in the database. 1 entries renamed:" ]]
|
||||
[[ ${lines[3]} = " from ./nonemptydirs/dir2/new-file-a.txt to ./nonemptydirs/dir2/new-file-a.txt2" ]]
|
||||
[[ ${lines[4]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
@test "bitrot detects delete files in a tree dir" {
|
||||
sleep 1
|
||||
rm nonemptydirs/dir2/new-file-a.txt2
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[0]} = "Checking bitrot.db integrity... ok." ]]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "1 entries in the database. 1 entries missing:" ]]
|
||||
[[ ${lines[3]} = " ./nonemptydirs/dir2/new-file-a.txt2" ]]
|
||||
[[ ${lines[4]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
|
||||
@test "bitrot detects new files and modified in a tree dir " {
|
||||
sleep 1
|
||||
touch more-files-{a,b,c,d,e,f,g}.txt
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "8 entries in the database. 7 entries new:" ]]
|
||||
[[ ${lines[3]} = " ./more-files-a.txt" ]]
|
||||
[[ ${lines[4]} = " ./more-files-b.txt" ]]
|
||||
[[ ${lines[5]} = " ./more-files-c.txt" ]]
|
||||
[[ ${lines[6]} = " ./more-files-d.txt" ]]
|
||||
[[ ${lines[7]} = " ./more-files-e.txt" ]]
|
||||
[[ ${lines[8]} = " ./more-files-f.txt" ]]
|
||||
[[ ${lines[9]} = " ./more-files-g.txt" ]]
|
||||
[[ ${lines[10]} = "1 entries updated:" ]]
|
||||
[[ ${lines[11]} = " ./nonemptydirs/dir2/new-file-b.txt" ]]
|
||||
[[ ${lines[12]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
@test "bitrot detects new files, modified, deleted and moved in a tree dir " {
|
||||
sleep 1
|
||||
for fil in {a,b,c,d,e,f,g}; do
|
||||
echo $RANDOM >> nonemptydirs/pl-more-files-$fil.txt
|
||||
done
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
mv more-files-a.txt more-files-a.txt2
|
||||
rm more-files-g.txt
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "14 entries in the database. 7 entries new:" ]]
|
||||
[[ ${lines[3]} = " ./nonemptydirs/pl-more-files-a.txt" ]]
|
||||
[[ ${lines[4]} = " ./nonemptydirs/pl-more-files-b.txt" ]]
|
||||
[[ ${lines[5]} = " ./nonemptydirs/pl-more-files-c.txt" ]]
|
||||
[[ ${lines[6]} = " ./nonemptydirs/pl-more-files-d.txt" ]]
|
||||
[[ ${lines[7]} = " ./nonemptydirs/pl-more-files-e.txt" ]]
|
||||
[[ ${lines[8]} = " ./nonemptydirs/pl-more-files-f.txt" ]]
|
||||
[[ ${lines[9]} = " ./nonemptydirs/pl-more-files-g.txt" ]]
|
||||
[[ ${lines[10]} = "1 entries updated:" ]]
|
||||
[[ ${lines[11]} = " ./nonemptydirs/dir2/new-file-b.txt" ]]
|
||||
[[ ${lines[12]} = "1 entries renamed:" ]]
|
||||
[[ ${lines[13]} = " from ./more-files-a.txt to ./more-files-a.txt2" ]]
|
||||
[[ ${lines[14]} = "1 entries missing:" ]]
|
||||
[[ ${lines[15]} = " ./more-files-g.txt" ]]
|
||||
[[ ${lines[16]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
|
||||
@test "bitrot detects new files, modified, deleted and moved in a tree dir 2" {
|
||||
sleep 1
|
||||
for fil in {a,b,c,d,e,f,g}; do
|
||||
echo $RANDOM >> nonemptydirs/pl2-more-files-$fil.txt
|
||||
done
|
||||
echo $RANDOM >> nonemptydirs/pl-more-files-a.txt
|
||||
mv nonemptydirs/pl-more-files-b.txt nonemptydirs/pl-more-files-b.txt2
|
||||
cp nonemptydirs/pl-more-files-g.txt nonemptydirs/pl2-more-files-g.txt2
|
||||
cp nonemptydirs/pl-more-files-d.txt nonemptydirs/pl2-more-files-d.txt2
|
||||
rm more-files-f.txt nonemptydirs/pl-more-files-c.txt
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "21 entries in the database. 9 entries new:" ]]
|
||||
[[ ${lines[3]} = " ./nonemptydirs/pl2-more-files-a.txt" ]]
|
||||
[[ ${lines[4]} = " ./nonemptydirs/pl2-more-files-b.txt" ]]
|
||||
[[ ${lines[5]} = " ./nonemptydirs/pl2-more-files-c.txt" ]]
|
||||
[[ ${lines[6]} = " ./nonemptydirs/pl2-more-files-d.txt" ]]
|
||||
[[ ${lines[7]} = " ./nonemptydirs/pl2-more-files-d.txt2" ]]
|
||||
[[ ${lines[8]} = " ./nonemptydirs/pl2-more-files-e.txt" ]]
|
||||
[[ ${lines[9]} = " ./nonemptydirs/pl2-more-files-f.txt" ]]
|
||||
[[ ${lines[10]} = " ./nonemptydirs/pl2-more-files-g.txt" ]]
|
||||
[[ ${lines[11]} = " ./nonemptydirs/pl2-more-files-g.txt2" ]]
|
||||
[[ ${lines[12]} = "1 entries updated:" ]]
|
||||
[[ ${lines[13]} = " ./nonemptydirs/pl-more-files-a.txt" ]]
|
||||
[[ ${lines[14]} = "1 entries renamed:" ]]
|
||||
[[ ${lines[15]} = " from ./nonemptydirs/pl-more-files-b.txt to ./nonemptydirs/pl-more-files-b.txt2" ]]
|
||||
[[ ${lines[16]} = "2 entries missing:" ]]
|
||||
[[ ${lines[17]} = " ./more-files-f.txt" ]]
|
||||
[[ ${lines[18]} = " ./nonemptydirs/pl-more-files-c.txt" ]]
|
||||
[[ ${lines[19]} = "Updating bitrot.sha512... done." ]]
|
||||
}
|
||||
|
||||
|
||||
@test "bitrot can operate with 3278 files easily in a dir (1)" {
|
||||
sleep 1
|
||||
mkdir -p alotfiles/here; cd alotfiles/here
|
||||
# create a 320KB file
|
||||
dd if=/dev/urandom of=masterfile bs=1 count=327680
|
||||
# split it in 3277 files (instantly) + masterfile = 3278
|
||||
split -b 100 -a 10 masterfile
|
||||
cd $test_dir
|
||||
run $cmd
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[2]} = "3299 entries in the database, 3278 new, 0 updated, 0 renamed, 0 missing." ]]
|
||||
}
|
||||
|
||||
@test "bitrot can operate with 3278 files easily in a dir (2)" {
|
||||
sleep 1
|
||||
mv alotfiles/here alotfiles/here-moved
|
||||
run $cmd
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[2]} = "3299 entries in the database, 0 new, 0 updated, 3278 renamed, 0 missing." ]]
|
||||
}
|
||||
|
||||
@test "bitrot can detect rotten bits in a dir (1)" {
|
||||
sleep 1
|
||||
touch non-rotten-file
|
||||
dd if=/dev/zero of=rotten-file bs=1k count=1000 &>/dev/null
|
||||
# let's make sure they share the same timestamp
|
||||
touch -r non-rotten-file rotten-file
|
||||
run $cmd -v
|
||||
|
||||
[ "$status" -eq 0 ]
|
||||
[[ ${lines[0]} = "Checking bitrot.db integrity... ok." ]]
|
||||
# [[ ${lines[1]} = "Finished. 0.00 MiB of data read. 0 errors found." ]]
|
||||
[[ ${lines[2]} = "3301 entries in the database, 2 entries new:" ]]
|
||||
[[ ${lines[3]} = " ./non-rotten-file" ]]
|
||||
[[ ${lines[4]} = " ./rotten-file" ]]
|
||||
}
|
||||
|
||||
@test "bitrot can detect rotten bits in a dir (2)" {
|
||||
sleep 1
|
||||
# modify the rotten file...
|
||||
dd if=/dev/urandom of=rotten-file bs=1k count=10 seek=1k conv=notrunc &>/dev/null
|
||||
# ...but revert the modification date
|
||||
touch -r non-rotten-file rotten-file
|
||||
run $cmd -q
|
||||
|
||||
[ "$status" -eq 1 ]
|
||||
[[ ${lines[0]} = *"error: SHA1 mismatch for ./rotten-file: expected"* ]]
|
||||
[[ ${lines[1]} = "error: There were 1 errors found." ]]
|
||||
}
|
||||
|
||||
@test "Clean everything" {
|
||||
run chmod -Rf a+w $test_dir
|
||||
run rm -rf $test_dir
|
||||
}
|
||||
@ -1,348 +0,0 @@
|
||||
"""
|
||||
NOTE: those tests are ordered and require pytest-order to run correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
TMP = Path("/tmp/")
|
||||
|
||||
|
||||
ReturnCode = int
|
||||
StdOut = list[str]
|
||||
StdErr = list[str]
|
||||
|
||||
|
||||
def bitrot(*args: str) -> tuple[ReturnCode, StdOut, StdErr]:
|
||||
cmd = [sys.executable, "-m", "bitrot"]
|
||||
cmd.extend(args)
|
||||
res = subprocess.run(shlex.join(cmd), shell=True, capture_output=True)
|
||||
stdout = (res.stdout or b"").decode("utf8")
|
||||
stderr = (res.stderr or b"").decode("utf8")
|
||||
return res.returncode, lines(stdout), lines(stderr)
|
||||
|
||||
|
||||
def bash(script, empty_dir: bool = False) -> bool:
|
||||
username = getpass.getuser()
|
||||
test_dir = TMP / f"bitrot-dir-{username}"
|
||||
if empty_dir and test_dir.is_dir():
|
||||
os.chdir(TMP)
|
||||
shutil.rmtree(test_dir)
|
||||
test_dir.mkdir(exist_ok=True)
|
||||
os.chdir(test_dir)
|
||||
|
||||
preamble = """
|
||||
set -euxo pipefail
|
||||
LC_ALL=en_US.UTF-8
|
||||
LANG=en_US.UTF-8
|
||||
"""
|
||||
|
||||
if script:
|
||||
# We need to wait a second for modification timestamps to differ so that
|
||||
# the ordering of the output stays the same every run of the tests.
|
||||
preamble += """
|
||||
sleep 1
|
||||
"""
|
||||
|
||||
script_path = TMP / "bitrot-test.bash"
|
||||
script_path.write_text(dedent(preamble + script))
|
||||
script_path.chmod(0o755)
|
||||
|
||||
out = subprocess.run(["bash", str(script_path)], capture_output=True)
|
||||
if out.returncode:
|
||||
print(f"Non-zero return code {out.returncode} when running {script_path}")
|
||||
if out.stdout:
|
||||
print(out.stdout)
|
||||
if out.stderr:
|
||||
print(out.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def lines(s: str) -> list[str]:
|
||||
r"""Only return non-empty lines that weren't killed by \r."""
|
||||
return [
|
||||
line.rstrip()
|
||||
for line in s.splitlines(keepends=True)
|
||||
if line and line.rstrip() and line[-1] != "\r"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.order(1)
|
||||
def test_command_exists() -> None:
|
||||
rc, out, err = bitrot("--help")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0].startswith("usage:")
|
||||
|
||||
assert bash("", empty_dir=True)
|
||||
|
||||
|
||||
@pytest.mark.order(2)
|
||||
def test_new_files_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
mkdir -p nonemptydirs/dir2/
|
||||
touch nonemptydirs/dir2/new-file-{a,b}.txt
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
# assert out[0] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[1] == "2 entries in the database. 2 entries new:"
|
||||
assert out[2] == " ./nonemptydirs/dir2/new-file-a.txt"
|
||||
assert out[3] == " ./nonemptydirs/dir2/new-file-b.txt"
|
||||
assert out[4] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(3)
|
||||
def test_modified_files_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-a.txt
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "2 entries in the database. 1 entries updated:"
|
||||
assert out[3] == " ./nonemptydirs/dir2/new-file-a.txt"
|
||||
assert out[4] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(4)
|
||||
def test_renamed_files_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
mv nonemptydirs/dir2/new-file-a.txt nonemptydirs/dir2/new-file-a.txt2
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "2 entries in the database. 1 entries renamed:"
|
||||
o3 = " from ./nonemptydirs/dir2/new-file-a.txt to ./nonemptydirs/dir2/new-file-a.txt2"
|
||||
assert out[3] == o3
|
||||
assert out[4] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(5)
|
||||
def test_deleted_files_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
rm nonemptydirs/dir2/new-file-a.txt2
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "1 entries in the database. 1 entries missing:"
|
||||
assert out[3] == " ./nonemptydirs/dir2/new-file-a.txt2"
|
||||
assert out[4] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(5)
|
||||
def test_new_files_and_modified_files_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
for fil in {a,b,c,d,e,f,g}; do
|
||||
echo $fil >> more-files-$fil.txt
|
||||
done
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "8 entries in the database. 7 entries new:"
|
||||
assert out[3] == " ./more-files-a.txt"
|
||||
assert out[4] == " ./more-files-b.txt"
|
||||
assert out[5] == " ./more-files-c.txt"
|
||||
assert out[6] == " ./more-files-d.txt"
|
||||
assert out[7] == " ./more-files-e.txt"
|
||||
assert out[8] == " ./more-files-f.txt"
|
||||
assert out[9] == " ./more-files-g.txt"
|
||||
assert out[10] == "1 entries updated:"
|
||||
assert out[11] == " ./nonemptydirs/dir2/new-file-b.txt"
|
||||
assert out[12] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(6)
|
||||
def test_new_files_modified_deleted_and_moved_in_a_tree_dir() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
for fil in {a,b,c,d,e,f,g}; do
|
||||
echo $fil $RANDOM >> nonemptydirs/pl-more-files-$fil.txt
|
||||
done
|
||||
echo $RANDOM >> nonemptydirs/dir2/new-file-b.txt
|
||||
mv more-files-a.txt more-files-a.txt2
|
||||
rm more-files-g.txt
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "14 entries in the database. 7 entries new:"
|
||||
assert out[3] == " ./nonemptydirs/pl-more-files-a.txt"
|
||||
assert out[4] == " ./nonemptydirs/pl-more-files-b.txt"
|
||||
assert out[5] == " ./nonemptydirs/pl-more-files-c.txt"
|
||||
assert out[6] == " ./nonemptydirs/pl-more-files-d.txt"
|
||||
assert out[7] == " ./nonemptydirs/pl-more-files-e.txt"
|
||||
assert out[8] == " ./nonemptydirs/pl-more-files-f.txt"
|
||||
assert out[9] == " ./nonemptydirs/pl-more-files-g.txt"
|
||||
assert out[10] == "1 entries updated:"
|
||||
assert out[11] == " ./nonemptydirs/dir2/new-file-b.txt"
|
||||
assert out[12] == "1 entries renamed:"
|
||||
assert out[13] == " from ./more-files-a.txt to ./more-files-a.txt2"
|
||||
assert out[14] == "1 entries missing:"
|
||||
assert out[15] == " ./more-files-g.txt"
|
||||
assert out[16] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(7)
|
||||
def test_new_files_modified_deleted_and_moved_in_a_tree_dir_2() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
for fil in {a,b,c,d,e,f,g}; do
|
||||
echo $RANDOM >> nonemptydirs/pl2-more-files-$fil.txt
|
||||
done
|
||||
echo $RANDOM >> nonemptydirs/pl-more-files-a.txt
|
||||
mv nonemptydirs/pl-more-files-b.txt nonemptydirs/pl-more-files-b.txt2
|
||||
cp nonemptydirs/pl-more-files-g.txt nonemptydirs/pl2-more-files-g.txt2
|
||||
cp nonemptydirs/pl-more-files-d.txt nonemptydirs/pl2-more-files-d.txt2
|
||||
rm more-files-f.txt nonemptydirs/pl-more-files-c.txt
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "21 entries in the database. 9 entries new:"
|
||||
assert out[3] == " ./nonemptydirs/pl2-more-files-a.txt"
|
||||
assert out[4] == " ./nonemptydirs/pl2-more-files-b.txt"
|
||||
assert out[5] == " ./nonemptydirs/pl2-more-files-c.txt"
|
||||
assert out[6] == " ./nonemptydirs/pl2-more-files-d.txt"
|
||||
assert out[7] == " ./nonemptydirs/pl2-more-files-d.txt2"
|
||||
assert out[8] == " ./nonemptydirs/pl2-more-files-e.txt"
|
||||
assert out[9] == " ./nonemptydirs/pl2-more-files-f.txt"
|
||||
assert out[10] == " ./nonemptydirs/pl2-more-files-g.txt"
|
||||
assert out[11] == " ./nonemptydirs/pl2-more-files-g.txt2"
|
||||
assert out[12] == "1 entries updated:"
|
||||
assert out[13] == " ./nonemptydirs/pl-more-files-a.txt"
|
||||
assert out[14] == "1 entries renamed:"
|
||||
o15 = " from ./nonemptydirs/pl-more-files-b.txt to ./nonemptydirs/pl-more-files-b.txt2"
|
||||
assert out[15] == o15
|
||||
assert out[16] == "2 entries missing:"
|
||||
assert out[17] == " ./more-files-f.txt"
|
||||
assert out[18] == " ./nonemptydirs/pl-more-files-c.txt"
|
||||
assert out[19] == "Updating bitrot.sha512... done."
|
||||
|
||||
|
||||
@pytest.mark.order(8)
|
||||
def test_3278_files() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
mkdir -p alotfiles/here; cd alotfiles/here
|
||||
# create a 320KB file
|
||||
dd if=/dev/urandom of=masterfile bs=1 count=327680
|
||||
# split it in 3277 files (instantly) + masterfile = 3278
|
||||
split -b 100 -a 10 masterfile
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot()
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
o2 = "3299 entries in the database, 3278 new, 0 updated, 0 renamed, 0 missing."
|
||||
assert out[2] == o2
|
||||
|
||||
|
||||
@pytest.mark.order(9)
|
||||
def test_3278_files_2() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
mv alotfiles/here alotfiles/here-moved
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot()
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
o2 = "3299 entries in the database, 0 new, 0 updated, 3278 renamed, 0 missing."
|
||||
assert out[2] == o2
|
||||
|
||||
|
||||
@pytest.mark.order(10)
|
||||
def test_rotten_file() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
touch non-rotten-file
|
||||
dd if=/dev/zero of=rotten-file bs=1k count=1000 &>/dev/null
|
||||
# let's make sure they share the same timestamp
|
||||
touch -r non-rotten-file rotten-file
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-v")
|
||||
assert rc == 0
|
||||
assert not err
|
||||
assert out[0] == "Checking bitrot.db integrity... ok."
|
||||
# assert out[1] == "Finished. 0.00 MiB of data read. 0 errors found."
|
||||
assert out[2] == "3301 entries in the database. 2 entries new:"
|
||||
assert out[3] == " ./non-rotten-file"
|
||||
assert out[4] == " ./rotten-file"
|
||||
|
||||
|
||||
@pytest.mark.order(11)
|
||||
def test_rotten_file_2() -> None:
|
||||
assert bash(
|
||||
"""
|
||||
# modify the rotten file...
|
||||
dd if=/dev/urandom of=rotten-file bs=1k count=10 seek=1k conv=notrunc &>/dev/null
|
||||
# ...but revert the modification date
|
||||
touch -r non-rotten-file rotten-file
|
||||
"""
|
||||
)
|
||||
rc, out, err = bitrot("-q")
|
||||
assert rc == 1
|
||||
assert not out
|
||||
e = (
|
||||
"error: SHA1 mismatch for ./rotten-file: expected"
|
||||
" 8fee1653e234fee8513245d3cb3e3c06d071493e, got"
|
||||
)
|
||||
assert err[0].startswith(e)
|
||||
assert err[1] == "error: There were 1 errors found."
|
||||
|
||||
|
||||
@pytest.mark.order("last")
|
||||
def test_cleanup() -> None:
|
||||
username = getpass.getuser()
|
||||
test_dir = TMP / f"bitrot-dir-{username}"
|
||||
if test_dir.is_dir():
|
||||
os.chdir(TMP)
|
||||
shutil.rmtree(test_dir)
|
||||
Loading…
x
Reference in New Issue
Block a user