pipe refs through unstructured parser

1 files changed, 30 insertions, 10 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 5e590a1..c6faece 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -172,6 +172,26 @@ class ReleaseExportExpanded(luigi.ExternalTask, Refcat):
 
 # ----8< Derivations
 
+class RefsWithUnstructured(Refcat):
+    """
+    Augment refs with data from unstructured. Do this first, so we can use it
+    all subsequent steps.
+    """
+    def requires(self):
+        return Refs()
+
+    def run(self):
+        output = shellout("""
+                          zstdcat -T0 {input} |
+                          skate-from-unstructured |
+                          zstd -T0 -c9 > {output}
+                          """,
+                          input=self.input().path)
+        luigi.LocalTarget(output).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+
 
 class ReleaseExportReduced(Refcat):
     """
@@ -224,7 +244,7 @@ class URLTabs(Refcat):
     sys     85m54.710s
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout("""
@@ -272,7 +292,7 @@ class RefsDOI(Refcat):
     TSV with (ident, doi, full doc).
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         """
@@ -305,7 +325,7 @@ class RefsPMID(Refcat):
     List of PMID, 74M refs seem to have one.
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout(r"""
@@ -329,7 +349,7 @@ class RefsPMCID(Refcat):
     List of PMCID.
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout(r"""
@@ -354,7 +374,7 @@ class RefsArxiv(Refcat):
     List of arxiv ids from refs.
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout(r"""
@@ -381,7 +401,7 @@ class RefsTitles(Refcat):
     RNA!editing!in!isolated!Physarum!mitochondria.!RNA*
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout(r"""
@@ -836,7 +856,7 @@ class RefsCounter(Refcat):
     Key counts, see: ref_counter.py.
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         counts = collections.Counter()
@@ -883,7 +903,7 @@ class RefsKeyStats(Refcat):
     How many titles, DOI, etc. do we have in refs?
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         stats = {
@@ -917,7 +937,7 @@ class RefsToRelease(Refcat):
     tools - XXX: polish.
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout("""
@@ -1290,7 +1310,7 @@ class UnmatchedRefs(Refcat):
     File with not yet considered refs (e.g. no title, doi, ...)
     """
     def requires(self):
-        return Refs()
+        return RefsWithUnstructured()
 
     def run(self):
         output = shellout("""