Post query json parse fix (#711)

* post append query: fix json parsing of lists to be identical to cdxj-indexer
if json parsing errors occur, log to stderr
fixes #709 in a better way

* update CHANGES.rst

CHANGES.rst

@@ -3,6 +3,7 @@ pywb 2.6.7 changelist
 
 * dependency: bump gevent to latest (21.12.0)
 * rewrite: fix eval rewriting where '._eval' was accidentally being rewritten
+* post-to-get conversion: properly handle json with top-level lists, to match cdxj-indexer, print parse errors, fixes `#709 <https://github.com/webrecorder/pywb/pull/709>`_
 
 pywb 2.6.6 changelist
 ~~~~~~~~~~~~~~~~~~~~~

pywb/warcserver/inputrequest.py

@@ -11,6 +11,7 @@ from io import BytesIO
 import base64
 import cgi
 import json
+import sys
 
 
 #=============================================================================
@@ -277,6 +278,7 @@ class MethodQueryCanonicalizer(object):
             try:
                 query = self.json_parse(query)
             except Exception as e:
+                sys.stderr.write("Ignoring query, error parsing as json: " + query.decode("utf-8") + "\n")
                 query = ''
 
         elif mime.startswith('text/plain'):
@@ -316,12 +318,17 @@ class MethodQueryCanonicalizer(object):
             dupes[n] += 1
             return n + "." + str(dupes[n]) + "_";
 
-        def _parser(dict_var):
-            for n, v in dict_var.items():
-                if isinstance(v, dict):
-                    _parser(v)
-                else:
-                    data[get_key(n)] = str(v)
+        def _parser(json_obj, name=""):
+            if isinstance(json_obj, dict):
+                for n, v in json_obj.items():
+                    _parser(v, n)
+
+            elif isinstance(json_obj, list):
+                for v in json_obj:
+                    _parser(v, name)
+
+            elif name:
+                data[get_key(name)] = str(json_obj)
 
         _parser(json.loads(string))
         return urlencode(data)