From 59b2cb65eafdba8fadba56d74089154863073d69 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Wed, 26 May 2021 23:38:33 +0200 Subject: tasks: fix pipeline --- python/refcat/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'python') diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index c45b6ed..5040fab 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -757,9 +757,10 @@ class OpenLibraryRelease(Refcat): output = shellout(""" zstdcat -T0 /magna/data/ol_dump_editions_latest.txt.zst | cut -f5 | - skate-conv -f oled -Xa <(zstdcat -T0 /magna/data/ol_author_mapping.tsv.zst) - """ - luigi.LocalTarget(output).move(self.output().path) + skate-conv -f oled -Xa <(zstdcat -T0 /magna/data/ol_author_mapping.tsv.zst) | + zstd -T0 -c > {output} + """) + luigi.LocalTarget(output).move(self.output().path def output(self): return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd) -- cgit v1.2.3