aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/refcat/tasks.py40
1 files changed, 30 insertions, 10 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 5e590a1..c6faece 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -172,6 +172,26 @@ class ReleaseExportExpanded(luigi.ExternalTask, Refcat):
# ----8< Derivations
+class RefsWithUnstructured(Refcat):
+ """
+ Augment refs with data from unstructured. Do this first, so we can use it
+ all subsequent steps.
+ """
+ def requires(self):
+ return Refs()
+
+ def run(self):
+ output = shellout("""
+ zstdcat -T0 {input} |
+ skate-from-unstructured |
+ zstd -T0 -c9 > {output}
+ """,
+ input=self.input().path)
+ luigi.LocalTarget(output).move(self.output().path)
+
+ def output(self):
+ return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+
class ReleaseExportReduced(Refcat):
"""
@@ -224,7 +244,7 @@ class URLTabs(Refcat):
sys 85m54.710s
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout("""
@@ -272,7 +292,7 @@ class RefsDOI(Refcat):
TSV with (ident, doi, full doc).
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
"""
@@ -305,7 +325,7 @@ class RefsPMID(Refcat):
List of PMID, 74M refs seem to have one.
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout(r"""
@@ -329,7 +349,7 @@ class RefsPMCID(Refcat):
List of PMCID.
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout(r"""
@@ -354,7 +374,7 @@ class RefsArxiv(Refcat):
List of arxiv ids from refs.
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout(r"""
@@ -381,7 +401,7 @@ class RefsTitles(Refcat):
RNA!editing!in!isolated!Physarum!mitochondria.!RNA*
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout(r"""
@@ -836,7 +856,7 @@ class RefsCounter(Refcat):
Key counts, see: ref_counter.py.
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
counts = collections.Counter()
@@ -883,7 +903,7 @@ class RefsKeyStats(Refcat):
How many titles, DOI, etc. do we have in refs?
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
stats = {
@@ -917,7 +937,7 @@ class RefsToRelease(Refcat):
tools - XXX: polish.
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout("""
@@ -1290,7 +1310,7 @@ class UnmatchedRefs(Refcat):
File with not yet considered refs (e.g. no title, doi, ...)
"""
def requires(self):
- return Refs()
+ return RefsWithUnstructured()
def run(self):
output = shellout("""