start to cleanup tasks

1 files changed, 17 insertions, 30 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 182a51f..4c6723c 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -139,7 +139,7 @@ class Refcat(BaseTask):
     BASE = settings.BASE
     TAG = 'refcat'
 
-    date = luigi.DateParameter(default=datetime.date(2021, 2, 20), description="a versioning help, change this manually")
+    date = luigi.DateParameter(default=datetime.date(2021, 5, 6), description="a versioning help, change this manually")
     tmpdir = luigi.Parameter(default="/magna/tmp", description="set tempdir", significant=False)
     n = luigi.IntParameter(default=multiprocessing.cpu_count(), significant=False)
 
@@ -156,7 +156,8 @@ class Refcat(BaseTask):
 
 class Refs(luigi.ExternalTask, Refcat):
     """
-    Compressed (zstd) references, as of 01/2021 containing ~1.8B docs.
+    Compressed (zstd) references, as of 01/2021 containing ~1.8B docs; this
+    might increase in a next version.
     """
     def output(self):
         return luigi.LocalTarget(path=settings.REFS_FILE, format=Zstd)
@@ -164,7 +165,7 @@ class Refs(luigi.ExternalTask, Refcat):
 
 class ReleaseExportExpanded(luigi.ExternalTask, Refcat):
     """
-    Release export, zstd version.
+    Fatcat release export, zstd version, from e.g. https://archive.org/details/fatcat_snapshots_and_exports
     """
     def output(self):
         return luigi.LocalTarget(path=settings.RELEASE_EXPORT_EXPANDED_FILE, format=Zstd)
@@ -180,9 +181,9 @@ class MAGPapers(luigi.ExternalTask, Refcat):
 
 class WikipediaCitationsMinimalDataset(luigi.ExternalTask, Refcat):
     """
-    From archive.org/details/wikipedia_citations_2020-07-14 (Wikipedia
+    From https://archive.org/details/wikipedia_citations_2020-07-14 (Wikipedia
     Citations: A comprehensive dataset of citations with identifiers extracted
-    from English Wikipedia).
+    from English Wikipedia); http://doi.org/10.5281/zenodo.3940692.
 
     Dataset contains parquet, but we want JSON here:
 
@@ -191,17 +192,22 @@ class WikipediaCitationsMinimalDataset(luigi.ExternalTask, Refcat):
     def output(self):
         return luigi.LocalTarget(path=os.path.join(settings.WIKIPEDIA_CITATIONS, "minimal_dataset.json"))
 
-class OpenLibraryDump(luigi.ExternalTask, Refcat):
 
+class OpenLibraryDump(luigi.ExternalTask, Refcat):
+    """
+    A solrdump exported version from a SOLR from:
+    https://archive.org/details/olsolr8-2021-04-12; about 30M items.
+    """
     def output(self):
         return luigi.LocalTarget(path=settings.OL_DUMP, format=Zstd)
 
+
 # ----8< Derivations
 
 
 class RefsWithUnstructured(Refcat):
     """
-    Augment refs with data from unstructured. Do this first, so we can use it
+    Augment refs with data from biblio.unstructured. Do this first, so we can use it
     all subsequent steps.
     """
     def requires(self):
@@ -211,7 +217,7 @@ class RefsWithUnstructured(Refcat):
         output = shellout("""
                           zstdcat -T0 {input} |
                           skate-from-unstructured |
-                          zstd -T0 -c9 > {output}
+                          zstd -T0 -c > {output}
                           """,
                           input=self.input().path)
         luigi.LocalTarget(output).move(self.output().path)
@@ -222,7 +228,7 @@ class RefsWithUnstructured(Refcat):
 
 class ReleaseExportReduced(Refcat):
     """
-    Reduce dataset size, stripping fields.
+    Reduce dataset size, stripping some heave fields.
     """
     def requires(self):
         return ReleaseExportExpanded()
@@ -240,25 +246,6 @@ class ReleaseExportReduced(Refcat):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
 
 
-class ReleaseExportTitleOnly(Refcat):
-    """
-    Reduce dataset size, only keep title.
-    """
-    def requires(self):
-        return ReleaseExportReduced()
-
-    def run(self):
-        output = shellout("""
-                          zstdcat -T0 {input} |
-                          parallel --block 10M -j 16 --pipe "jq -rc '{{\"title\": .title}}'" |
-                          zstd -T0 -c9 > {output}
-                          """,
-                          input=self.input().path)
-        luigi.LocalTarget(output).move(self.output().path)
-
-    def output(self):
-        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
-
 
 class URLTabs(Refcat):
     """
@@ -1431,8 +1418,10 @@ class RefsSortedIdent(Refcat):
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
 
+
 # OL
 
+
 class WithISBN(Refcat):
     """
     Keeps converted refs with isbn.
@@ -1475,5 +1464,3 @@ class OpenLibraryWorks(Refcat):
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
-
-