From 043771ab253926e411f0465fd83be4e2636d95c9 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:23:34 -0800 Subject: [PATCH 01/27] initial commit, setting up for py 2 & 3 --- tests/test_dump-anydbm.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/test_dump-anydbm.py diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py new file mode 100644 index 0000000..1c5bac2 --- /dev/null +++ b/tests/test_dump-anydbm.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +#from warcprox.bin import dump-anydbm +import pytest + +# will try as python 3 then default to python 2 modules +try: + import dbm + ndbm = dbm.ndbm + gdbm = dbm.gdbm + dumb = dbm.dumb + whichdb = dbm.whichdb +except: + import dbm as ndbm + import gdbm + import dumbdbm as dumb + from whichdb import whichdb + + + From 2dd147c1c259ef5d7868a8e3041d23335ac58cb8 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:25:24 -0800 Subject: [PATCH 02/27] create first fixture for gdbm --- tests/test_dump-anydbm.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 1c5bac2..ee27a59 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -16,5 +16,12 @@ except: import dumbdbm as dumb from whichdb import whichdb - - +@pytest.fixture +def make_gdbm_test_db(): + db_name ="test_gdbm" + print "creating", db_name + test_db = gdbm.open(db_name, "n") + test_db['very first key'] = 'very first value' + test_db['second key'] = 'second value' + test_db.close() + return db_name From acc55ce266398931717cf0fac379e33b7740d88f Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:26:00 -0800 Subject: [PATCH 03/27] create first test for gdbm fixture --- tests/test_dump-anydbm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index ee27a59..a0c93c7 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -25,3 +25,7 @@ def make_gdbm_test_db(): test_db['second key'] = 'second value' test_db.close() return db_name + +def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): + print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" + assert whichdb(make_gdbm_test_db) == "dbm.gdbm" or "gdbm" \ No newline at end of file From 7991640291d0f7941ff7711948b0a5fb4a6bdfd9 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:27:14 -0800 Subject: [PATCH 04/27] take advantage of teardown of fixture to delete temp db file --- tests/test_dump-anydbm.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index a0c93c7..d35deef 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -2,6 +2,7 @@ #from warcprox.bin import dump-anydbm import pytest +import os # will try as python 3 then default to python 2 modules try: @@ -16,14 +17,21 @@ except: import dumbdbm as dumb from whichdb import whichdb -@pytest.fixture -def make_gdbm_test_db(): +@pytest.fixture(scope = "function") +def make_gdbm_test_db(request): db_name ="test_gdbm" print "creating", db_name test_db = gdbm.open(db_name, "n") test_db['very first key'] = 'very first value' test_db['second key'] = 'second value' test_db.close() + def delete_test_dumbdbm(): + print "deleting", db_name + os.remove(db_name+".dir") + os.remove(db_name+".bak") + os.remove(db_name+".dat") + + request.addfinalizer(delete_test_dumbdbm) return db_name def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): From 41836c86f8eaaaa4df68adb6949349eade91c679 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:30:31 -0800 Subject: [PATCH 05/27] added test to confirm correct number of keys in db, key exists, and key has correct value --- tests/test_dump-anydbm.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index d35deef..5ea245a 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -36,4 +36,11 @@ def make_gdbm_test_db(request): def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_gdbm_test_db) == "dbm.gdbm" or "gdbm" \ No newline at end of file + assert whichdb(make_gdbm_test_db) == "dbm.gdbm" or "gdbm" + +def test_assert_reading_gdbm_correctly(make_gdbm_test_db): + print "running assert_reading_gdbm_correctly with gdbm test db" + db = gdbm.open(make_gdbm_test_db, "r") + assert len(db.keys()) == 2 + assert db.has_key('very first key') + assert db['very first key'] == 'very first value' \ No newline at end of file From 1ea6eb5b9a0f3725254157c4e7a8fdcf17f08337 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:33:31 -0800 Subject: [PATCH 06/27] add dubmdbm fixture and recognition test --- tests/test_dump-anydbm.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 5ea245a..b1923d5 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -34,6 +34,23 @@ def make_gdbm_test_db(request): request.addfinalizer(delete_test_dumbdbm) return db_name +@pytest.fixture(scope = "function") +def make_dumbdbm_test_db(request): + db_name ="test_dumbdbm" + print "creating", db_name + test_db = dumb.open(db_name, "n") + test_db['very first key'] = 'very first value' + test_db['second key'] = 'second value' + test_db.close() + def delete_test_dumbdbm(): + print "deleting", db_name + os.remove(db_name+".dir") + os.remove(db_name+".bak") + os.remove(db_name+".dat") + + request.addfinalizer(delete_test_dumbdbm) + return db_name+".dir" + def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" assert whichdb(make_gdbm_test_db) == "dbm.gdbm" or "gdbm" @@ -43,4 +60,8 @@ def test_assert_reading_gdbm_correctly(make_gdbm_test_db): db = gdbm.open(make_gdbm_test_db, "r") assert len(db.keys()) == 2 assert db.has_key('very first key') - assert db['very first key'] == 'very first value' \ No newline at end of file + assert db['very first key'] == 'very first value' + +def test_assert_dumbdbm_db_is_created_and_correctly_identified(make_dumbdbm_test_db): + print "runing assert_dumbdbm_db_is_created_and_correctly_identified with gdbm test file" + assert whichdb(make_dumbdbm_test_db) == "dbm.dumb" or "dumbdbm" \ No newline at end of file From 0f48f6bed9aee77828b932b57d3c248701980a62 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:38:45 -0800 Subject: [PATCH 07/27] removed the mixed remove statement from dumbdbm in gdbm and used the appropriate filename remove --- tests/test_dump-anydbm.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index b1923d5..a7e99af 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -27,10 +27,8 @@ def make_gdbm_test_db(request): test_db.close() def delete_test_dumbdbm(): print "deleting", db_name - os.remove(db_name+".dir") - os.remove(db_name+".bak") - os.remove(db_name+".dat") - + os.remove(db_name) + request.addfinalizer(delete_test_dumbdbm) return db_name From 24a3437fda9f5f7ca402b963bd17dd51472f8693 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 14:48:26 -0800 Subject: [PATCH 08/27] adapted reading for dumbdbm. also removed the '.dir' from the name being passed through. Not needed for whichdb or .open --- tests/test_dump-anydbm.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index a7e99af..0497fb8 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -47,7 +47,7 @@ def make_dumbdbm_test_db(request): os.remove(db_name+".dat") request.addfinalizer(delete_test_dumbdbm) - return db_name+".dir" + return db_name def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" @@ -62,4 +62,14 @@ def test_assert_reading_gdbm_correctly(make_gdbm_test_db): def test_assert_dumbdbm_db_is_created_and_correctly_identified(make_dumbdbm_test_db): print "runing assert_dumbdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_dumbdbm_test_db) == "dbm.dumb" or "dumbdbm" \ No newline at end of file + assert whichdb(make_dumbdbm_test_db) == "dbm.dumb" or "dumbdbm" + +def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): + print "running assert_reading_dumbdbm_correctly with dumbdbm test db" + db = dumb.open(make_dumbdbm_test_db, "r") + assert len(db.keys()) == 2 + assert db.has_key('very first key') + assert db['very first key'] == 'very first value' + + + From efb96ee0c3bef458665872079352f6e8b5eea532 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 15:35:23 -0800 Subject: [PATCH 09/27] added test for dump-anydbm that uses module subprocess to run script --- tests/test_dump-anydbm.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 0497fb8..0e2ddee 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -3,6 +3,7 @@ #from warcprox.bin import dump-anydbm import pytest import os +import subprocess # to access the script from shell # will try as python 3 then default to python 2 modules try: @@ -71,5 +72,16 @@ def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): assert db.has_key('very first key') assert db['very first key'] == 'very first value' +def test_dumpanydbm_identify_gbdm(make_gdbm_test_db): + print "running test_dumpanydbm_identify_gbdm" + output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) + output = output.split("/n") + + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print which + assert which == "gdbm" or "dbm.gdbm" + + From 4718cd785389d63475013a236b1cc54bd2e634be Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 16:04:26 -0800 Subject: [PATCH 10/27] updated keys & vals to global predefined, added db type specification (for the differences between py2 & 3), finished up test for dump-anydbm with gdbm --- tests/test_dump-anydbm.py | 55 +++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 0e2ddee..b53ad87 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -11,13 +11,30 @@ try: ndbm = dbm.ndbm gdbm = dbm.gdbm dumb = dbm.dumb + whichdb = dbm.whichdb + + ndbm_type = "dbm.ndbm" + gdbm_type = "dbm.gdbm" + dumb_type = "dbm.dumb" + except: import dbm as ndbm import gdbm import dumbdbm as dumb + from whichdb import whichdb + ndbm_type = "dbm" + gdbm_type = "gdbm" + dumb_type = "dumbdbm" + +#global settings +key1 = 'very first key' +key2 = 'second key' +val1 = 'very first value' +val2 = 'second value' + @pytest.fixture(scope = "function") def make_gdbm_test_db(request): db_name ="test_gdbm" @@ -38,8 +55,8 @@ def make_dumbdbm_test_db(request): db_name ="test_dumbdbm" print "creating", db_name test_db = dumb.open(db_name, "n") - test_db['very first key'] = 'very first value' - test_db['second key'] = 'second value' + test_db[key1] = val1 + test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): print "deleting", db_name @@ -50,37 +67,53 @@ def make_dumbdbm_test_db(request): request.addfinalizer(delete_test_dumbdbm) return db_name +# def test_fixture(make_ndbm_test_db): +# print "runing test_fixture with" +# assert whichdb(make_ndbm_test_db) == "dbm" + def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_gdbm_test_db) == "dbm.gdbm" or "gdbm" + assert whichdb(make_gdbm_test_db) == gdbm_type def test_assert_reading_gdbm_correctly(make_gdbm_test_db): print "running assert_reading_gdbm_correctly with gdbm test db" db = gdbm.open(make_gdbm_test_db, "r") assert len(db.keys()) == 2 - assert db.has_key('very first key') - assert db['very first key'] == 'very first value' + assert db.has_key(key1) + assert db[key1] == val1 def test_assert_dumbdbm_db_is_created_and_correctly_identified(make_dumbdbm_test_db): print "runing assert_dumbdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_dumbdbm_test_db) == "dbm.dumb" or "dumbdbm" + assert whichdb(make_dumbdbm_test_db) == dumb_type def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): print "running assert_reading_dumbdbm_correctly with dumbdbm test db" db = dumb.open(make_dumbdbm_test_db, "r") assert len(db.keys()) == 2 - assert db.has_key('very first key') - assert db['very first key'] == 'very first value' + assert db.has_key(key1) + assert db[key1] == val1 def test_dumpanydbm_identify_gbdm(make_gdbm_test_db): print "running test_dumpanydbm_identify_gbdm" output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) - output = output.split("/n") - + output = output.strip().split("\n") + assert len(output) == 3 # 2 keys plus whichdb line + # split on space, then grab 4th word, which is db type which = output[0].split(' ')[3] print which - assert which == "gdbm" or "dbm.gdbm" + assert which == gdbm_type + + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 + + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 + + From c6cbda1187f1071bf5b32dfb50e6ab44a811ae5c Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 16:50:35 -0800 Subject: [PATCH 11/27] updated test for dumbdbm. dump-anydbm cannot run any dubmdbm, the test now reflects that --- tests/test_dump-anydbm.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index b53ad87..4d568ff 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -40,13 +40,13 @@ def make_gdbm_test_db(request): db_name ="test_gdbm" print "creating", db_name test_db = gdbm.open(db_name, "n") - test_db['very first key'] = 'very first value' - test_db['second key'] = 'second value' + test_db[key1] = val1 + test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): print "deleting", db_name os.remove(db_name) - + request.addfinalizer(delete_test_dumbdbm) return db_name @@ -93,8 +93,8 @@ def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): assert db.has_key(key1) assert db[key1] == val1 -def test_dumpanydbm_identify_gbdm(make_gdbm_test_db): - print "running test_dumpanydbm_identify_gbdm" +def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): + print "running test_dumpanydbm_identify_gdbm" output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) output = output.strip().split("\n") assert len(output) == 3 # 2 keys plus whichdb line @@ -113,8 +113,11 @@ def test_dumpanydbm_identify_gbdm(make_gdbm_test_db): assert db_dump_second_pair[0] == key2 assert db_dump_second_pair[1] == val2 - - +def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): + print "running test_dumpanydbm_identify_dumbdbm" + output = subprocess.call(["dump-anydbm", make_dumbdbm_test_db]) + # output = output.strip().split("\n") + assert output != 0 # unable to read dumbdbm, so dump-anydbm exits with error From 99d077b89c5a1321bedf8c8bf865078b3f48d26c Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 17 Dec 2013 17:29:50 -0800 Subject: [PATCH 12/27] still having trouble gettting the ndbm module to correctly make an ndbm instead of bsd db. this fixture setup seems to work, tho bogglingly. It creates 2 files, one ndbm & the other bsd db. requires further investigation --- tests/test_dump-anydbm.py | 50 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 4d568ff..d71384e 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -50,6 +50,22 @@ def make_gdbm_test_db(request): request.addfinalizer(delete_test_dumbdbm) return db_name +@pytest.fixture(scope = "function") +def make_ndbm_test_db(request): + db_name = "test_ndbm" + print "creating", db_name + open(db_name+".db", 'a').close() + test_db = ndbm.open(db_name+".db", "n") + test_db['very first key'] = 'very first value' + test_db['second key'] = 'second value' + test_db.close() + def delete_test_ndbm(): + print "deleting", db_name + # os.remove(db_name+".db") + + request.addfinalizer(delete_test_ndbm) + return db_name+".db" + @pytest.fixture(scope = "function") def make_dumbdbm_test_db(request): db_name ="test_dumbdbm" @@ -67,9 +83,11 @@ def make_dumbdbm_test_db(request): request.addfinalizer(delete_test_dumbdbm) return db_name -# def test_fixture(make_ndbm_test_db): -# print "runing test_fixture with" -# assert whichdb(make_ndbm_test_db) == "dbm" + + +def test_fixture(make_ndbm_test_db): + print "runing test_fixture with" + assert whichdb(make_ndbm_test_db) == ndbm_type def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" @@ -93,6 +111,9 @@ def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): assert db.has_key(key1) assert db[key1] == val1 + +######################### + def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): print "running test_dumpanydbm_identify_gdbm" output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) @@ -113,6 +134,29 @@ def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): assert db_dump_second_pair[0] == key2 assert db_dump_second_pair[1] == val2 +def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): + print "running test_dumpanydbm_identify_ndbm" + output = subprocess.check_output(["dump-anydbm", make_ndbm_test_db]) + output = output.strip().split("\n") + + print output + + assert len(output) == 3 # 2 keys plus whichdb line + + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print which + assert which == ndbm_type + + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 + + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 + def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): print "running test_dumpanydbm_identify_dumbdbm" output = subprocess.call(["dump-anydbm", make_dumbdbm_test_db]) From d6cee750bc73f05b72413e0f96bca73fe2684c2d Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Wed, 18 Dec 2013 18:23:15 -0800 Subject: [PATCH 13/27] updating the test fixture for ndbm. whichdb needs the filename w/o '.db' to correctly identify and dbm.open does not the extension to read it --- tests/test_dump-anydbm.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index d71384e..0a5617d 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -54,17 +54,16 @@ def make_gdbm_test_db(request): def make_ndbm_test_db(request): db_name = "test_ndbm" print "creating", db_name - open(db_name+".db", 'a').close() - test_db = ndbm.open(db_name+".db", "n") + test_db = ndbm.open(db_name, "n") test_db['very first key'] = 'very first value' test_db['second key'] = 'second value' test_db.close() def delete_test_ndbm(): print "deleting", db_name - # os.remove(db_name+".db") + os.remove(db_name+".db") request.addfinalizer(delete_test_ndbm) - return db_name+".db" + return db_name @pytest.fixture(scope = "function") def make_dumbdbm_test_db(request): From 43158fdce2c1564936d7b1dafbfa384f19a89b5a Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Wed, 18 Dec 2013 21:16:52 -0800 Subject: [PATCH 14/27] reconfigured dump-anydbm to check the output of whichdb instead of checking for file existence first. This helps bypass the issue of an ndbm database only be recognized without its extension. If which returns None, then the file does not exist, and if it returns '' then it cannot process it --- bin/dump-anydbm | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bin/dump-anydbm b/bin/dump-anydbm index f1f8a42..dce0dc8 100755 --- a/bin/dump-anydbm +++ b/bin/dump-anydbm @@ -24,15 +24,31 @@ if __name__ == "__main__": sys.stderr.write("usage: {} DBM_FILE\n".format(sys.argv[0])) exit(1) - if not os.path.exists(sys.argv[1]): + filename = sys.argv[1] + file_location = sys.argv[1] + which = whichdb(filename) + + # if which returns none and the file does not exist, print usage line + if which == None and not os.path.exists(sys.argv[1]): sys.stderr.write('No such file {}\n\n'.format(sys.argv[1])) sys.stderr.write("usage: {} DBM_FILE\n".format(sys.argv[0])) exit(1) - which = whichdb(sys.argv[1]) - print('{} is a {} db'.format(sys.argv[1], which)) + # covers case where an ndbm is checked with its extension & identified incorrectly + elif 'bsd' in which: + correct_file = filename.split(".db")[0] + correct_which = whichdb(correct_file) + if correct_which == ('dbm' or 'dbm.ndbm'): + filename = correct_file + which = correct_which - db = dbm.open(sys.argv[1], 'r') + elif which == '': + sys.stderr.write("{} is an unrecognized database type\n".format(sys.argv[1])) + exit(1) + + print('{} is a {} db'.format(filename, which)) + + db = dbm.open(filename, 'r') for key in db.keys(): print("{}:{}".format(key, db[key])) From f8549f69cc1aac66276fa2e20a597afd5928a471 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Wed, 18 Dec 2013 21:17:41 -0800 Subject: [PATCH 15/27] cleaned up tests and removed unnecessary testing of fixtures. Only test of dump-anydbm remain --- tests/test_dump-anydbm.py | 59 +++++++++++++-------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index 0a5617d..c8fc8a5 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -#from warcprox.bin import dump-anydbm import pytest import os import subprocess # to access the script from shell @@ -55,8 +54,8 @@ def make_ndbm_test_db(request): db_name = "test_ndbm" print "creating", db_name test_db = ndbm.open(db_name, "n") - test_db['very first key'] = 'very first value' - test_db['second key'] = 'second value' + test_db[key1] = val1 + test_db[key2] = val2 test_db.close() def delete_test_ndbm(): print "deleting", db_name @@ -82,37 +81,6 @@ def make_dumbdbm_test_db(request): request.addfinalizer(delete_test_dumbdbm) return db_name - - -def test_fixture(make_ndbm_test_db): - print "runing test_fixture with" - assert whichdb(make_ndbm_test_db) == ndbm_type - -def test_assert_gdbm_db_is_created_and_correctly_identified(make_gdbm_test_db): - print "runing assert_gdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_gdbm_test_db) == gdbm_type - -def test_assert_reading_gdbm_correctly(make_gdbm_test_db): - print "running assert_reading_gdbm_correctly with gdbm test db" - db = gdbm.open(make_gdbm_test_db, "r") - assert len(db.keys()) == 2 - assert db.has_key(key1) - assert db[key1] == val1 - -def test_assert_dumbdbm_db_is_created_and_correctly_identified(make_dumbdbm_test_db): - print "runing assert_dumbdbm_db_is_created_and_correctly_identified with gdbm test file" - assert whichdb(make_dumbdbm_test_db) == dumb_type - -def test_assert_reading_dumbdbm_correctly(make_dumbdbm_test_db): - print "running assert_reading_dumbdbm_correctly with dumbdbm test db" - db = dumb.open(make_dumbdbm_test_db, "r") - assert len(db.keys()) == 2 - assert db.has_key(key1) - assert db[key1] == val1 - - -######################### - def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): print "running test_dumpanydbm_identify_gdbm" output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) @@ -138,8 +106,6 @@ def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): output = subprocess.check_output(["dump-anydbm", make_ndbm_test_db]) output = output.strip().split("\n") - print output - assert len(output) == 3 # 2 keys plus whichdb line # split on space, then grab 4th word, which is db type @@ -158,9 +124,22 @@ def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): print "running test_dumpanydbm_identify_dumbdbm" - output = subprocess.call(["dump-anydbm", make_dumbdbm_test_db]) - # output = output.strip().split("\n") - assert output != 0 # unable to read dumbdbm, so dump-anydbm exits with error - + output = subprocess.check_output(["dump-anydbm", make_dumbdbm_test_db]) + output = output.strip().split("\n") + assert len(output) == 3 # 2 keys plus whichdb line + + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print which + assert which == dumb_type + + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 + + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 From 44ea56d65cbe31af35cccf3f5c695c43691851d1 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Fri, 20 Dec 2013 13:43:49 -0800 Subject: [PATCH 16/27] addressed odd python 3 issue where whichdb cannot recognize an ndbm unless that module has been previously imported. Also removed file_location variable as it wasn't being used afterall --- bin/dump-anydbm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/dump-anydbm b/bin/dump-anydbm index dce0dc8..83e1872 100755 --- a/bin/dump-anydbm +++ b/bin/dump-anydbm @@ -10,7 +10,9 @@ deduplication database or a playback index database, but it is a generic tool. try: import dbm + from dbm import ndbm whichdb = dbm.whichdb + except: import anydbm dbm = anydbm @@ -25,7 +27,6 @@ if __name__ == "__main__": exit(1) filename = sys.argv[1] - file_location = sys.argv[1] which = whichdb(filename) # if which returns none and the file does not exist, print usage line From 48a2db509f4c4cf8370fa34cc0a21d88a7a9e8fd Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Fri, 20 Dec 2013 13:47:41 -0800 Subject: [PATCH 17/27] updated string printing to be consistent between python 2 & 3. Previously, python 3 would print the db key & values as byte code, while python 2 inherently ignores this when printing. Now the stdout is the same in both versions. --- bin/dump-anydbm | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/dump-anydbm b/bin/dump-anydbm index 83e1872..689c9ca 100755 --- a/bin/dump-anydbm +++ b/bin/dump-anydbm @@ -45,11 +45,17 @@ if __name__ == "__main__": elif which == '': sys.stderr.write("{} is an unrecognized database type\n".format(sys.argv[1])) + sys.stderr.write("Try the file again by removing the extension\n") exit(1) - print('{} is a {} db'.format(filename, which)) + try: + out = sys.stdout.buffer + + except AttributeError: + out = sys.stdout + + out.write(filename.encode('UTF-8') + b' is a ' + which.encode('UTF-8') + b' db\n') db = dbm.open(filename, 'r') - for key in db.keys(): - print("{}:{}".format(key, db[key])) + out.write(key + b":" + db[key] + b"\n") From 6a129fed34d2bf8fbc7338e39a9e17d23e6accd1 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Fri, 20 Dec 2013 13:51:34 -0800 Subject: [PATCH 18/27] corrected the import of python 3 modules. Updated print statements to function form to work correctly with python 2 & 3. Also decoded the subprocess output into unicode string for splicing and processing of keys --- tests/test_dump-anydbm.py | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_dump-anydbm.py b/tests/test_dump-anydbm.py index c8fc8a5..b23fc44 100644 --- a/tests/test_dump-anydbm.py +++ b/tests/test_dump-anydbm.py @@ -7,14 +7,14 @@ import subprocess # to access the script from shell # will try as python 3 then default to python 2 modules try: import dbm - ndbm = dbm.ndbm - gdbm = dbm.gdbm - dumb = dbm.dumb + from dbm import ndbm + from dbm import gnu as gdbm + from dbm import dumb whichdb = dbm.whichdb ndbm_type = "dbm.ndbm" - gdbm_type = "dbm.gdbm" + gdbm_type = "dbm.gnu" dumb_type = "dbm.dumb" except: @@ -33,17 +33,18 @@ key1 = 'very first key' key2 = 'second key' val1 = 'very first value' val2 = 'second value' +dump_anydbm = "dump-anydbm" @pytest.fixture(scope = "function") def make_gdbm_test_db(request): db_name ="test_gdbm" - print "creating", db_name + print("creating", db_name) test_db = gdbm.open(db_name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): - print "deleting", db_name + print("deleting", db_name) os.remove(db_name) request.addfinalizer(delete_test_dumbdbm) @@ -52,13 +53,13 @@ def make_gdbm_test_db(request): @pytest.fixture(scope = "function") def make_ndbm_test_db(request): db_name = "test_ndbm" - print "creating", db_name + print("creating", db_name) test_db = ndbm.open(db_name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_ndbm(): - print "deleting", db_name + print("deleting", db_name) os.remove(db_name+".db") request.addfinalizer(delete_test_ndbm) @@ -67,13 +68,13 @@ def make_ndbm_test_db(request): @pytest.fixture(scope = "function") def make_dumbdbm_test_db(request): db_name ="test_dumbdbm" - print "creating", db_name + print("creating", db_name) test_db = dumb.open(db_name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): - print "deleting", db_name + print("deleting", db_name) os.remove(db_name+".dir") os.remove(db_name+".bak") os.remove(db_name+".dat") @@ -82,14 +83,14 @@ def make_dumbdbm_test_db(request): return db_name def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): - print "running test_dumpanydbm_identify_gdbm" - output = subprocess.check_output(["dump-anydbm", make_gdbm_test_db]) - output = output.strip().split("\n") + print("running test_dumpanydbm_identify_gdbm") + output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) + output = output.decode(encoding = 'UTF-8').strip().split("\n") assert len(output) == 3 # 2 keys plus whichdb line # split on space, then grab 4th word, which is db type which = output[0].split(' ')[3] - print which + print(which) assert which == gdbm_type #split remaining lines on ':' that separates key & value @@ -102,15 +103,15 @@ def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): assert db_dump_second_pair[1] == val2 def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): - print "running test_dumpanydbm_identify_ndbm" - output = subprocess.check_output(["dump-anydbm", make_ndbm_test_db]) - output = output.strip().split("\n") + print("running test_dumpanydbm_identify_ndbm") + output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) + output = output.decode(encoding = 'UTF-8').strip().split("\n") assert len(output) == 3 # 2 keys plus whichdb line # split on space, then grab 4th word, which is db type which = output[0].split(' ')[3] - print which + print(which) assert which == ndbm_type #split remaining lines on ':' that separates key & value @@ -123,14 +124,15 @@ def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): assert db_dump_second_pair[1] == val2 def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): - print "running test_dumpanydbm_identify_dumbdbm" - output = subprocess.check_output(["dump-anydbm", make_dumbdbm_test_db]) - output = output.strip().split("\n") + print("running test_dumpanydbm_identify_dumbdbm") + output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) + output = output.decode(encoding = 'UTF-8').strip().split("\n") assert len(output) == 3 # 2 keys plus whichdb line + print(output) # split on space, then grab 4th word, which is db type which = output[0].split(' ')[3] - print which + print(which) assert which == dumb_type #split remaining lines on ':' that separates key & value @@ -141,5 +143,3 @@ def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): db_dump_second_pair = output[2].split(':') assert db_dump_second_pair[0] == key2 assert db_dump_second_pair[1] == val2 - - From d643be1c8c7c5c4bae11bd8deabc3741fb5abdec Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Fri, 20 Dec 2013 14:01:42 -0800 Subject: [PATCH 19/27] moved dump-anydbm test file to be in the existing test folder, as proximity to dump-anydbm script is not necessary --- {tests => warcprox/tests}/test_dump-anydbm.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tests => warcprox/tests}/test_dump-anydbm.py (100%) diff --git a/tests/test_dump-anydbm.py b/warcprox/tests/test_dump-anydbm.py similarity index 100% rename from tests/test_dump-anydbm.py rename to warcprox/tests/test_dump-anydbm.py From e6606c67fa7bfb52b431fa82e2987d4027c0a48c Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 31 Dec 2013 12:34:09 -0800 Subject: [PATCH 20/27] committing updates to tox.ini file for use with tox --- tox.ini | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 2403045..8f74e7d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,11 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py32, py33 +envlist = py33 [testenv] -commands = {envpython} setup.py test - +commands = py.test +deps = + pytest + warcprox + requests From 39fb1c75ba1ff511c1857ac52aa384bcc576a9bd Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Tue, 31 Dec 2013 13:47:50 -0800 Subject: [PATCH 21/27] added py27 & py32 back into envlist for tox to run against --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8f74e7d..1d8f4f4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py33 +envlist = py27, py32, py33 [testenv] commands = py.test From 4b0ab0ff728f8bd70581c84bf7c8d68c52448f80 Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Thu, 2 Jan 2014 16:29:15 -0800 Subject: [PATCH 22/27] updated file to PEP 8, as editor was complaining, and tabs are generally bad --- warcprox/tests/test_dump-anydbm.py | 209 +++++++++++++++-------------- 1 file changed, 109 insertions(+), 100 deletions(-) diff --git a/warcprox/tests/test_dump-anydbm.py b/warcprox/tests/test_dump-anydbm.py index b23fc44..fbe9a96 100644 --- a/warcprox/tests/test_dump-anydbm.py +++ b/warcprox/tests/test_dump-anydbm.py @@ -5,28 +5,28 @@ import os import subprocess # to access the script from shell # will try as python 3 then default to python 2 modules -try: - import dbm - from dbm import ndbm - from dbm import gnu as gdbm - from dbm import dumb +try: + import dbm + from dbm import ndbm + from dbm import gnu as gdbm + from dbm import dumb - whichdb = dbm.whichdb + whichdb = dbm.whichdb - ndbm_type = "dbm.ndbm" - gdbm_type = "dbm.gnu" - dumb_type = "dbm.dumb" + ndbm_type = "dbm.ndbm" + gdbm_type = "dbm.gnu" + dumb_type = "dbm.dumb" except: - import dbm as ndbm - import gdbm - import dumbdbm as dumb + import dbm as ndbm + import gdbm + import dumbdbm as dumb - from whichdb import whichdb + from whichdb import whichdb - ndbm_type = "dbm" - gdbm_type = "gdbm" - dumb_type = "dumbdbm" + ndbm_type = "dbm" + gdbm_type = "gdbm" + dumb_type = "dumbdbm" #global settings key1 = 'very first key' @@ -35,111 +35,120 @@ val1 = 'very first value' val2 = 'second value' dump_anydbm = "dump-anydbm" -@pytest.fixture(scope = "function") + +@pytest.fixture(scope="function") def make_gdbm_test_db(request): - db_name ="test_gdbm" - print("creating", db_name) - test_db = gdbm.open(db_name, "n") - test_db[key1] = val1 - test_db[key2] = val2 - test_db.close() - def delete_test_dumbdbm(): - print("deleting", db_name) - os.remove(db_name) + db_name = "test_gdbm" + print("creating", db_name) + test_db = gdbm.open(db_name, "n") + test_db[key1] = val1 + test_db[key2] = val2 + test_db.close() - request.addfinalizer(delete_test_dumbdbm) - return db_name + def delete_test_dumbdbm(): + print("deleting", db_name) + os.remove(db_name) -@pytest.fixture(scope = "function") + request.addfinalizer(delete_test_dumbdbm) + return db_name + + +@pytest.fixture(scope="function") def make_ndbm_test_db(request): - db_name = "test_ndbm" - print("creating", db_name) - test_db = ndbm.open(db_name, "n") - test_db[key1] = val1 - test_db[key2] = val2 - test_db.close() - def delete_test_ndbm(): - print("deleting", db_name) - os.remove(db_name+".db") + db_name = "test_ndbm" + print("creating", db_name) + test_db = ndbm.open(db_name, "n") + test_db[key1] = val1 + test_db[key2] = val2 + test_db.close() - request.addfinalizer(delete_test_ndbm) - return db_name + def delete_test_ndbm(): + print("deleting", db_name) + os.remove(db_name + ".db") -@pytest.fixture(scope = "function") + request.addfinalizer(delete_test_ndbm) + return db_name + + +@pytest.fixture(scope="function") def make_dumbdbm_test_db(request): - db_name ="test_dumbdbm" - print("creating", db_name) - test_db = dumb.open(db_name, "n") - test_db[key1] = val1 - test_db[key2] = val2 - test_db.close() - def delete_test_dumbdbm(): - print("deleting", db_name) - os.remove(db_name+".dir") - os.remove(db_name+".bak") - os.remove(db_name+".dat") + db_name = "test_dumbdbm" + print("creating", db_name) + test_db = dumb.open(db_name, "n") + test_db[key1] = val1 + test_db[key2] = val2 + test_db.close() + + def delete_test_dumbdbm(): + print("deleting", db_name) + os.remove(db_name + ".dir") + os.remove(db_name + ".bak") + os.remove(db_name + ".dat") + + request.addfinalizer(delete_test_dumbdbm) + return db_name - request.addfinalizer(delete_test_dumbdbm) - return db_name def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): - print("running test_dumpanydbm_identify_gdbm") - output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) - output = output.decode(encoding = 'UTF-8').strip().split("\n") - assert len(output) == 3 # 2 keys plus whichdb line + print("running test_dumpanydbm_identify_gdbm") + output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) + output = output.decode(encoding='UTF-8').strip().split("\n") + assert len(output) == 3 # 2 keys plus whichdb line - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == gdbm_type + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print(which) + assert which == gdbm_type - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 + + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): - print("running test_dumpanydbm_identify_ndbm") - output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) - output = output.decode(encoding = 'UTF-8').strip().split("\n") + print("running test_dumpanydbm_identify_ndbm") + output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) + output = output.decode(encoding='UTF-8').strip().split("\n") - assert len(output) == 3 # 2 keys plus whichdb line + assert len(output) == 3 # 2 keys plus whichdb line - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == ndbm_type + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print(which) + assert which == ndbm_type - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 + + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): - print("running test_dumpanydbm_identify_dumbdbm") - output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) - output = output.decode(encoding = 'UTF-8').strip().split("\n") - assert len(output) == 3 # 2 keys plus whichdb line - print(output) + print("running test_dumpanydbm_identify_dumbdbm") + output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) + output = output.decode(encoding='UTF-8').strip().split("\n") + assert len(output) == 3 # 2 keys plus whichdb line + print(output) - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == dumb_type + # split on space, then grab 4th word, which is db type + which = output[0].split(' ')[3] + print(which) + assert which == dumb_type - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 + #split remaining lines on ':' that separates key & value + db_dump_first_pair = output[1].split(':') + assert db_dump_first_pair[0] == key1 + assert db_dump_first_pair[1] == val1 - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 + db_dump_second_pair = output[2].split(':') + assert db_dump_second_pair[0] == key2 + assert db_dump_second_pair[1] == val2 From 1b69aea7ed6e09f25912a078d2dcccd1ec8f3c0e Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Thu, 2 Jan 2014 17:05:45 -0800 Subject: [PATCH 23/27] removed the string splicing and replaced with one clear assert statement based on the script output for each test. simplifies and clarifies the test --- warcprox/tests/test_dump-anydbm.py | 56 +++++------------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/warcprox/tests/test_dump-anydbm.py b/warcprox/tests/test_dump-anydbm.py index fbe9a96..7aabcf6 100644 --- a/warcprox/tests/test_dump-anydbm.py +++ b/warcprox/tests/test_dump-anydbm.py @@ -92,63 +92,25 @@ def make_dumbdbm_test_db(request): def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): print("running test_dumpanydbm_identify_gdbm") output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) - output = output.decode(encoding='UTF-8').strip().split("\n") - assert len(output) == 3 # 2 keys plus whichdb line + print(b"script printout: \n" + output) - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == gdbm_type - - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 - - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 + assert (output == b'test_gdbm is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == b'test_gdbm is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): print("running test_dumpanydbm_identify_ndbm") output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) - output = output.decode(encoding='UTF-8').strip().split("\n") + print(b"script printout: \n" + output) - assert len(output) == 3 # 2 keys plus whichdb line - - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == ndbm_type - - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 - - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 + assert (output == b'test_ndbm is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == b'test_ndbm is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): print("running test_dumpanydbm_identify_dumbdbm") output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) - output = output.decode(encoding='UTF-8').strip().split("\n") - assert len(output) == 3 # 2 keys plus whichdb line - print(output) + print(b"script printout: \n" + output) - # split on space, then grab 4th word, which is db type - which = output[0].split(' ')[3] - print(which) - assert which == dumb_type - - #split remaining lines on ':' that separates key & value - db_dump_first_pair = output[1].split(':') - assert db_dump_first_pair[0] == key1 - assert db_dump_first_pair[1] == val1 - - db_dump_second_pair = output[2].split(':') - assert db_dump_second_pair[0] == key2 - assert db_dump_second_pair[1] == val2 + assert (output == b'test_dumbdbm is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == b'test_dumbdbm is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') From b6ea681c2bd2d90a30d8faf59c42c7ffd2ef432c Mon Sep 17 00:00:00 2001 From: Kelsey Hawley Date: Thu, 2 Jan 2014 18:18:46 -0800 Subject: [PATCH 24/27] changed file creation and deletion to use temporaryfile. Still needed to use os to delete the 'extra' files that ndbm & dumbdbm created. Also did not explicitly state the file name in checking the output statements, as now they are random everytime. --- warcprox/tests/test_dump-anydbm.py | 55 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/warcprox/tests/test_dump-anydbm.py b/warcprox/tests/test_dump-anydbm.py index 7aabcf6..b402fd5 100644 --- a/warcprox/tests/test_dump-anydbm.py +++ b/warcprox/tests/test_dump-anydbm.py @@ -2,6 +2,7 @@ import pytest import os +import tempfile import subprocess # to access the script from shell # will try as python 3 then default to python 2 modules @@ -38,55 +39,57 @@ dump_anydbm = "dump-anydbm" @pytest.fixture(scope="function") def make_gdbm_test_db(request): - db_name = "test_gdbm" - print("creating", db_name) - test_db = gdbm.open(db_name, "n") + print("creating test gdbm file") + temp_file = tempfile.NamedTemporaryFile() + test_db = gdbm.open(temp_file.name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): - print("deleting", db_name) - os.remove(db_name) + print("deleting test gdbm file") + temp_file.close() request.addfinalizer(delete_test_dumbdbm) - return db_name + return temp_file.name @pytest.fixture(scope="function") def make_ndbm_test_db(request): - db_name = "test_ndbm" - print("creating", db_name) - test_db = ndbm.open(db_name, "n") + print("creating test ndbm file") + temp_file = tempfile.NamedTemporaryFile() + test_db = ndbm.open(temp_file.name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_ndbm(): - print("deleting", db_name) - os.remove(db_name + ".db") + print("deleting test ndbm file") + temp_file.close() + os.remove(temp_file.name + ".db") request.addfinalizer(delete_test_ndbm) - return db_name + return temp_file.name @pytest.fixture(scope="function") def make_dumbdbm_test_db(request): - db_name = "test_dumbdbm" - print("creating", db_name) - test_db = dumb.open(db_name, "n") + print("creating test dumbdbm file") + temp_file = tempfile.NamedTemporaryFile() + test_db = dumb.open(temp_file.name, "n") test_db[key1] = val1 test_db[key2] = val2 test_db.close() def delete_test_dumbdbm(): - print("deleting", db_name) - os.remove(db_name + ".dir") - os.remove(db_name + ".bak") - os.remove(db_name + ".dat") + print("deleting test dumbdbm file") + temp_file.close() + os.remove(temp_file.name + ".dir") + os.remove(temp_file.name + ".bak") + os.remove(temp_file.name + ".dat") request.addfinalizer(delete_test_dumbdbm) - return db_name + return temp_file.name def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): @@ -94,8 +97,8 @@ def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) print(b"script printout: \n" + output) - assert (output == b'test_gdbm is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == b'test_gdbm is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == make_gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == make_gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): @@ -103,8 +106,8 @@ def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) print(b"script printout: \n" + output) - assert (output == b'test_ndbm is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == b'test_ndbm is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == make_ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == make_ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): @@ -112,5 +115,5 @@ def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) print(b"script printout: \n" + output) - assert (output == b'test_dumbdbm is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == b'test_dumbdbm is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == make_dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == make_dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') From e89fba433284b69951cdee668283c4682a528ef7 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 6 Jan 2014 17:22:00 -0800 Subject: [PATCH 25/27] logic fix --- bin/dump-anydbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dump-anydbm b/bin/dump-anydbm index 689c9ca..1d1ae4b 100755 --- a/bin/dump-anydbm +++ b/bin/dump-anydbm @@ -39,7 +39,7 @@ if __name__ == "__main__": elif 'bsd' in which: correct_file = filename.split(".db")[0] correct_which = whichdb(correct_file) - if correct_which == ('dbm' or 'dbm.ndbm'): + if correct_which in ('dbm', 'dbm.ndbm'): filename = correct_file which = correct_which From b4a7a7fa5f4c1e4daaa49d78e4811809a0ab1be5 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 6 Jan 2014 17:22:36 -0800 Subject: [PATCH 26/27] remove warcprox from deps, because it downloads and installs from pypi instead of using local checkout --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1d8f4f4..8bdedc6 100644 --- a/tox.ini +++ b/tox.ini @@ -10,5 +10,4 @@ envlist = py27, py32, py33 commands = py.test deps = pytest - warcprox requests From f69ec424fbb6022a9328a3e9aef165d439ccc549 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 6 Jan 2014 17:22:49 -0800 Subject: [PATCH 27/27] minor cleanup --- warcprox/tests/test_dump-anydbm.py | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/warcprox/tests/test_dump-anydbm.py b/warcprox/tests/test_dump-anydbm.py index b402fd5..1fddefc 100644 --- a/warcprox/tests/test_dump-anydbm.py +++ b/warcprox/tests/test_dump-anydbm.py @@ -14,9 +14,9 @@ try: whichdb = dbm.whichdb - ndbm_type = "dbm.ndbm" - gdbm_type = "dbm.gnu" - dumb_type = "dbm.dumb" + ndbm_type = b"dbm.ndbm" + gdbm_type = b"dbm.gnu" + dumb_type = b"dbm.dumb" except: import dbm as ndbm @@ -25,9 +25,9 @@ except: from whichdb import whichdb - ndbm_type = "dbm" - gdbm_type = "gdbm" - dumb_type = "dumbdbm" + ndbm_type = b"dbm" + gdbm_type = b"gdbm" + dumb_type = b"dumbdbm" #global settings key1 = 'very first key' @@ -38,7 +38,7 @@ dump_anydbm = "dump-anydbm" @pytest.fixture(scope="function") -def make_gdbm_test_db(request): +def gdbm_test_db(request): print("creating test gdbm file") temp_file = tempfile.NamedTemporaryFile() test_db = gdbm.open(temp_file.name, "n") @@ -46,16 +46,16 @@ def make_gdbm_test_db(request): test_db[key2] = val2 test_db.close() - def delete_test_dumbdbm(): + def delete_gdbm_test_db(): print("deleting test gdbm file") temp_file.close() - request.addfinalizer(delete_test_dumbdbm) + request.addfinalizer(delete_gdbm_test_db) return temp_file.name @pytest.fixture(scope="function") -def make_ndbm_test_db(request): +def ndbm_test_db(request): print("creating test ndbm file") temp_file = tempfile.NamedTemporaryFile() test_db = ndbm.open(temp_file.name, "n") @@ -73,7 +73,7 @@ def make_ndbm_test_db(request): @pytest.fixture(scope="function") -def make_dumbdbm_test_db(request): +def dumbdbm_test_db(request): print("creating test dumbdbm file") temp_file = tempfile.NamedTemporaryFile() test_db = dumb.open(temp_file.name, "n") @@ -81,39 +81,39 @@ def make_dumbdbm_test_db(request): test_db[key2] = val2 test_db.close() - def delete_test_dumbdbm(): + def delete_dumbdbm_test_db(): print("deleting test dumbdbm file") temp_file.close() os.remove(temp_file.name + ".dir") os.remove(temp_file.name + ".bak") os.remove(temp_file.name + ".dat") - request.addfinalizer(delete_test_dumbdbm) + request.addfinalizer(delete_dumbdbm_test_db) return temp_file.name -def test_dumpanydbm_identify_gdbm(make_gdbm_test_db): +def test_dumpanydbm_identify_gdbm(gdbm_test_db): print("running test_dumpanydbm_identify_gdbm") - output = subprocess.check_output([dump_anydbm, make_gdbm_test_db]) + output = subprocess.check_output([dump_anydbm, gdbm_test_db]) print(b"script printout: \n" + output) - assert (output == make_gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == make_gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == gdbm_test_db.encode(encoding='UTF-8') + b' is a ' + gdbm_type + b' db\nsecond key:second value\nvery first key:very first value\n') -def test_dumpanydbm_identify_ndbm(make_ndbm_test_db): +def test_dumpanydbm_identify_ndbm(ndbm_test_db): print("running test_dumpanydbm_identify_ndbm") - output = subprocess.check_output([dump_anydbm, make_ndbm_test_db]) + output = subprocess.check_output([dump_anydbm, ndbm_test_db]) print(b"script printout: \n" + output) - assert (output == make_ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == make_ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == ndbm_test_db.encode(encoding='UTF-8') + b' is a ' + ndbm_type + b' db\nsecond key:second value\nvery first key:very first value\n') -def test_dumpanydbm_identify_dumbdbm(make_dumbdbm_test_db): +def test_dumpanydbm_identify_dumbdbm(dumbdbm_test_db): print("running test_dumpanydbm_identify_dumbdbm") - output = subprocess.check_output([dump_anydbm, make_dumbdbm_test_db]) + output = subprocess.check_output([dump_anydbm, dumbdbm_test_db]) print(b"script printout: \n" + output) - assert (output == make_dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nvery first key:very first value\nsecond key:second value\n' or - output == make_dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type.encode(encoding='UTF-8') + b' db\nsecond key:second value\nvery first key:very first value\n') + assert (output == dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type + b' db\nvery first key:very first value\nsecond key:second value\n' or + output == dumbdbm_test_db.encode(encoding='UTF-8') + b' is a ' + dumb_type + b' db\nsecond key:second value\nvery first key:very first value\n')