diff options
| -rw-r--r-- | python/refcat/techreport.py | 22 |
1 files changed, 21 insertions, 1 deletions
diff --git a/python/refcat/techreport.py b/python/refcat/techreport.py index e23e151..0b1faa5 100644 --- a/python/refcat/techreport.py +++ b/python/refcat/techreport.py @@ -2,7 +2,7 @@ Tasks for techreport. """ import luigi -from refcat.tasks import Refcat, OpenCitations +from refcat.tasks import Refcat, OpenCitations, BrefCombined from refcat.base import shellout, Zstd @@ -27,3 +27,23 @@ class COCIDOIOnly(Refcat): def output(self): return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + + +class BrefDOIOnly(Refcat): + """ + Bref, reduced to doi, so we can compare with others. + """ + def requires(self): + return BrefWithDOI() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + parallel --pipe -j 16 --block 10M "jq -rc '[.source_doi, .target_doi] | @csv'" | + perl -F, -lane 'printf qq[%s\n], join ",", sort @F' | + zstd -c -T0 > {output} + """, + input=self.input().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) |