[0.9.0] Python 3 compat, --quiet obeyed for bitrot.db checksum checks

This commit is contained in:
Lukasz Langa 2016-08-09 14:51:57 -07:00
parent bfb73acc70
commit 5ed89d8b1a
3 changed files with 54 additions and 35 deletions

View File

@ -36,6 +36,13 @@ a 100 GB Aperture library in under 10 minutes. Both tests on HFS+.
Change Log Change Log
---------- ----------
0.9.0
~~~~~
* bugfix: bitrot.db checksum checking messages now obey --quiet
* Python 3 compatibility
0.8.0 0.8.0
~~~~~ ~~~~~

View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2013 by Łukasz Langa # Copyright (C) 2013 by Łukasz Langa
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights # in the Software without restriction, including without limitation the rights
@ -21,15 +21,13 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE. # THE SOFTWARE.
import codecs
import os import os
import sys import sys
from setuptools import setup, find_packages from setuptools import setup, find_packages
reload(sys)
sys.setdefaultencoding('utf8')
current_dir = os.path.abspath(os.path.dirname(__file__)) current_dir = os.path.abspath(os.path.dirname(__file__))
ld_file = open(os.path.join(current_dir, 'README.rst')) ld_file = codecs.open(os.path.join(current_dir, 'README.rst'), encoding='utf8')
try: try:
long_description = ld_file.read() long_description = ld_file.read()
finally: finally:
@ -44,7 +42,7 @@ release = ".".join(str(num) for num in VERSION)
setup( setup(
name = 'bitrot', name = 'bitrot',
version = release, version = release,
author = 'Łukasz Langa', author = u'Łukasz Langa',
author_email = 'lukasz@langa.pl', author_email = 'lukasz@langa.pl',
description = ("Detects bit rotten files on the hard drive to save your " description = ("Detects bit rotten files on the hard drive to save your "
"precious photo and music collection from slow decay."), "precious photo and music collection from slow decay."),
@ -63,11 +61,11 @@ setup(
], ],
classifiers = [ classifiers = [
'Development Status :: 3 - Alpha', 'Development Status :: 4 - Beta',
'License :: OSI Approved :: MIT License', 'License :: OSI Approved :: MIT License',
'Natural Language :: English', 'Natural Language :: English',
'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 2 :: Only', 'Programming Language :: Python :: 3',
'Programming Language :: Python', 'Programming Language :: Python',
'Topic :: System :: Filesystems', 'Topic :: System :: Filesystems',
'Topic :: System :: Monitoring', 'Topic :: System :: Monitoring',

View File

@ -42,11 +42,16 @@ import time
DEFAULT_CHUNK_SIZE = 16384 DEFAULT_CHUNK_SIZE = 16384
DOT_THRESHOLD = 200 DOT_THRESHOLD = 200
VERSION = (0, 8, 0) VERSION = (0, 9, 0)
IGNORED_FILE_SYSTEM_ERRORS = {errno.ENOENT, errno.EACCES} IGNORED_FILE_SYSTEM_ERRORS = {errno.ENOENT, errno.EACCES}
FSENCODING = sys.getfilesystemencoding() FSENCODING = sys.getfilesystemencoding()
if sys.version[0] == '2':
str = type(u'text')
# use `bytes` for bytestrings
def sha1(path, chunk_size): def sha1(path, chunk_size):
digest = hashlib.sha1() digest = hashlib.sha1()
with open(path, 'rb') as f: with open(path, 'rb') as f:
@ -62,6 +67,7 @@ def ts():
def get_sqlite3_cursor(path, copy=False): def get_sqlite3_cursor(path, copy=False):
path = path.decode(FSENCODING)
if copy: if copy:
if not os.path.exists(path): if not os.path.exists(path):
raise ValueError("error: bitrot database at {} does not exist." raise ValueError("error: bitrot database at {} does not exist."
@ -162,7 +168,7 @@ class Bitrot(object):
self._last_commit_ts = time.time() self._last_commit_ts = time.time()
def run(self): def run(self):
check_sha512_integrity() check_sha512_integrity(verbosity=self.verbosity)
bitrot_db = get_path() bitrot_db = get_path()
bitrot_sha512 = get_path(ext=b'sha512') bitrot_sha512 = get_path(ext=b'sha512')
@ -278,7 +284,7 @@ class Bitrot(object):
missing_paths, missing_paths,
) )
update_sha512_integrity() update_sha512_integrity(verbosity=self.verbosity)
if errors: if errors:
raise BitrotException( raise BitrotException(
@ -340,7 +346,7 @@ class Bitrot(object):
print(' ', path) print(' ', path)
if not any((new_paths, updated_paths, missing_paths)): if not any((new_paths, updated_paths, missing_paths)):
print() print()
if self.test: if self.test and self.verbosity:
print('warning: database file not updated on disk (test mode).') print('warning: database file not updated on disk (test mode).')
def handle_unknown_path(self, cur, new_path, new_mtime, new_sha1): def handle_unknown_path(self, cur, new_path, new_mtime, new_sha1):
@ -396,43 +402,48 @@ def stable_sum(bitrot_db):
return digest.hexdigest() return digest.hexdigest()
def check_sha512_integrity(): def check_sha512_integrity(verbosity=1):
sha512_path = get_path(ext='sha512') sha512_path = get_path(ext=b'sha512')
if not os.path.exists(sha512_path): if not os.path.exists(sha512_path):
return return
print('Checking bitrot.db integrity... ', end='') if verbosity:
print('Checking bitrot.db integrity... ', end='')
sys.stdout.flush()
with open(sha512_path, 'rb') as f: with open(sha512_path, 'rb') as f:
old_sha512 = f.read().strip() old_sha512 = f.read().strip()
bitrot_db = get_path() bitrot_db = get_path()
digest = hashlib.sha512() digest = hashlib.sha512()
with open(bitrot_db, 'rb') as f: with open(bitrot_db, 'rb') as f:
digest.update(f.read()) digest.update(f.read())
new_sha512 = digest.hexdigest() new_sha512 = digest.hexdigest().encode('ascii')
if new_sha512 != old_sha512: if new_sha512 != old_sha512:
if len(old_sha512) == 128: if verbosity:
if len(old_sha512) == 128:
print(
"error: SHA512 of the file is different, bitrot.db might "
"be corrupt.",
)
else:
print(
"error: SHA512 of the file is different but bitrot.sha512 "
"has a suspicious length. It might be corrupt.",
)
print( print(
"error: SHA512 of the file is different, bitrot.db might be " "If you'd like to continue anyway, delete the .bitrot.sha512 "
"corrupt." "file and try again.",
file=sys.stderr,
) )
else:
print(
"error: SHA512 of the file is different but bitrot.sha512 has "
"a suspicious length. It might be corrupt."
)
print(
"If you'd like to continue anyway, delete the .bitrot.sha512 "
"file and try again."
)
raise BitrotException( raise BitrotException(
3, 'bitrot.db integrity check failed, cannot continue.', 3, 'bitrot.db integrity check failed, cannot continue.',
) )
print('ok.') if verbosity:
print('ok.')
def update_sha512_integrity(): def update_sha512_integrity(verbosity=1):
old_sha512 = 0 old_sha512 = 0
sha512_path = get_path(ext='sha512') sha512_path = get_path(ext=b'sha512')
if os.path.exists(sha512_path): if os.path.exists(sha512_path):
with open(sha512_path, 'rb') as f: with open(sha512_path, 'rb') as f:
old_sha512 = f.read().strip() old_sha512 = f.read().strip()
@ -440,12 +451,15 @@ def update_sha512_integrity():
digest = hashlib.sha512() digest = hashlib.sha512()
with open(bitrot_db, 'rb') as f: with open(bitrot_db, 'rb') as f:
digest.update(f.read()) digest.update(f.read())
new_sha512 = digest.hexdigest() new_sha512 = digest.hexdigest().encode('ascii')
if new_sha512 != old_sha512: if new_sha512 != old_sha512:
print('Updating bitrot.sha512... ', end='') if verbosity:
print('Updating bitrot.sha512... ', end='')
sys.stdout.flush()
with open(sha512_path, 'wb') as f: with open(sha512_path, 'wb') as f:
f.write(new_sha512) f.write(new_sha512)
print('done.') if verbosity:
print('done.')
def run_from_command_line(): def run_from_command_line():
global FSENCODING global FSENCODING
@ -492,7 +506,7 @@ def run_from_command_line():
try: try:
print(stable_sum()) print(stable_sum())
except RuntimeError as e: except RuntimeError as e:
print(unicode(e).encode('utf8'), file=sys.stderr) print(str(e).encode('utf8'), file=sys.stderr)
else: else:
verbosity = 1 verbosity = 1
if args.quiet: if args.quiet: