add dedupe and query-fatcat commands

1 files changed, 27 insertions, 0 deletions

diff --git a/fatcat_covid19/dedupe.py b/fatcat_covid19/dedupe.py
new file mode 100644
index 0000000..86cf8fa
--- /dev/null
+++ b/fatcat_covid19/dedupe.py
@@ -0,0 +1,27 @@
+
+import sys
+import json
+import datetime
+
+
+def dedupe_file(json_input, json_output):
+    """
+    Takes JSON file of "fatcat enriched" content, and de-dupes based on the
+    fatcat identifier.
+    """
+    rows = dict()
+    for l in json_input:
+        l = json.loads(l)
+        key = l.get('release_id')
+        if not key:
+            continue
+        if not key in rows:
+            rows[key] = l
+            continue
+        for other_info in ['cord19_paper', 'fatcat_hit',]:
+            if other_info in l:
+                rows[key][other_info] = l[other_info]
+
+    for k in rows:
+        print(json.dumps(rows[k], sort_keys=True), file=json_output)
+