diff options
| -rw-r--r-- | python/refcat/tasks.py | 31 |
1 files changed, 31 insertions, 0 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 90c3073..319cdcd 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -1247,6 +1247,37 @@ class BrefSortedByWorkID(Refcat): return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd) +class RawRefsMapped(Refcat): + """ + Key raw refs by work id. Since data is already sorted by work id, this can + skip the sorting step. + """ + def requires(self): + return Refs() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + skate-map -m ff -x work_ident | + zstd -c -T0 > {output} + """, + input=self.input().path) + luigi.LocalTarget(tmpf).move(self.output().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + + +class BrefCombined(Refcat): + """ + Merge the raw references with our biblioref format, such that + we include all non-matched items and also consider duplicates. + + This is basically a reduce step, where we group by work id (since the raw + refs were already sorted by work id). + """ + + # # Extra # ----- |