tasks: add RawRefsMapped

1 files changed, 31 insertions, 0 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 90c3073..319cdcd 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -1247,6 +1247,37 @@ class BrefSortedByWorkID(Refcat):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
 
 
+class RawRefsMapped(Refcat):
+    """
+    Key raw refs by work id. Since data is already sorted by work id, this can
+    skip the sorting step.
+    """
+    def requires(self):
+        return Refs()
+
+    def run(self):
+        output = shellout("""
+                          zstdcat -T0 {input} |
+                          skate-map -m ff -x work_ident |
+                          zstd -c -T0 > {output}
+                          """,
+                          input=self.input().path)
+        luigi.LocalTarget(tmpf).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
+
+
+class BrefCombined(Refcat):
+    """
+    Merge the raw references with our biblioref format, such that
+    we include all non-matched items and also consider duplicates.
+
+    This is basically a reduce step, where we group by work id (since the raw
+    refs were already sorted by work id).
+    """
+
+
 #
 # Extra
 # -----