From 6043cb86de54b4fceaabbf1a561b600d2f3d244f Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Wed, 8 Sep 2021 12:37:58 +0200 Subject: tasks: add BrefDOIOnly --- python/refcat/techreport.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/python/refcat/techreport.py b/python/refcat/techreport.py index e23e151..0b1faa5 100644 --- a/python/refcat/techreport.py +++ b/python/refcat/techreport.py @@ -2,7 +2,7 @@ Tasks for techreport. """ import luigi -from refcat.tasks import Refcat, OpenCitations +from refcat.tasks import Refcat, OpenCitations, BrefCombined from refcat.base import shellout, Zstd @@ -27,3 +27,23 @@ class COCIDOIOnly(Refcat): def output(self): return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + + +class BrefDOIOnly(Refcat): + """ + Bref, reduced to doi, so we can compare with others. + """ + def requires(self): + return BrefWithDOI() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + parallel --pipe -j 16 --block 10M "jq -rc '[.source_doi, .target_doi] | @csv'" | + perl -F, -lane 'printf qq[%s\n], join ",", sort @F' | + zstd -c -T0 > {output} + """, + input=self.input().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) -- cgit v1.2.3