From ad9ba8f8b52490ef528a2c9dd188ba6f491ed0c7 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Thu, 10 Jun 2021 12:01:25 +0200 Subject: tasks: add RawRefsMapped --- python/refcat/tasks.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'python') diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 90c3073..319cdcd 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -1247,6 +1247,37 @@ class BrefSortedByWorkID(Refcat): return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd) +class RawRefsMapped(Refcat): + """ + Key raw refs by work id. Since data is already sorted by work id, this can + skip the sorting step. + """ + def requires(self): + return Refs() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + skate-map -m ff -x work_ident | + zstd -c -T0 > {output} + """, + input=self.input().path) + luigi.LocalTarget(tmpf).move(self.output().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + + +class BrefCombined(Refcat): + """ + Merge the raw references with our biblioref format, such that + we include all non-matched items and also consider duplicates. + + This is basically a reduce step, where we group by work id (since the raw + refs were already sorted by work id). + """ + + # # Extra # ----- -- cgit v1.2.3