diff options
| author | Martin Czygan <martin.czygan@gmail.com> | 2021-05-11 19:50:53 +0200 |
|---|---|---|
| committer | Martin Czygan <martin.czygan@gmail.com> | 2021-05-11 19:50:53 +0200 |
| commit | da83d4584c8f131cb3ad80ed2928a9fd033af5f9 (patch) | |
| tree | a3935087dd8815ca953ea2fd24740a1e1f4a7670 /python | |
| parent | 4016f2b50bf7b22eeb9eb41cf83d07bf59e8d7b3 (diff) | |
| download | refcat-da83d4584c8f131cb3ad80ed2928a9fd033af5f9.tar.gz refcat-da83d4584c8f131cb3ad80ed2928a9fd033af5f9.zip | |
rename: skate-to-doi to skate-cleanup
Diffstat (limited to 'python')
| -rw-r--r-- | python/refcat/tasks.py | 25 |
1 files changed, 24 insertions, 1 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 915d406..f21fade 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -295,6 +295,29 @@ class URLTabs(Refcat): def output(self): return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) +class URLList(Refcat): + """ + List of cleaned URLs from refs. + """ + def requires(self): + return URLList() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + cut -f 3 | + skate-cleanup -c url -B -S -f 1 | + LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 | + zstd -T0 -c > {output} + """, + n=self.n, + tmpdir=self.tmpdir, + input=self.input().path) + luigi.LocalTarget(output).move(self.output().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + class RefsDOI(Refcat): """ Sorted (doi, doc) tuples from refs. 225m48.755s @@ -306,7 +329,7 @@ class RefsDOI(Refcat): output = shellout(""" zstdcat -T0 {input} | skate-map -m ff -x biblio.doi -skip-on-empty 1 | - skate-to-doi -f 1 | + skate-cleanup -c doi -f 1 | LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 | zstd -T0 -c > {output} """, |