tasks: cleanup

1 files changed, 28 insertions, 39 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index aaa0551..54a5890 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -186,6 +186,22 @@ class OpenLibraryDump(luigi.ExternalTask, Refcat):
         return luigi.LocalTarget(path=settings.OL_DUMP, format=Zstd)
 
 
+class OpenLibraryEditions(luigi.ExternalTask, Refcat):
+    """
+    Editions file.
+    """
+    def output(self):
+        return luigi.LocalTarget(path=settings.OL_DUMP_EDITIONS, format=Zstd)
+
+
+class OpenLibraryWorks(luigi.ExternalTask, Refcat):
+    """
+    Works file.
+    """
+    def output(self):
+        return luigi.LocalTarget(path=settings.OL_DUMP_WORKS, format=Zstd)
+
+
 # ----8< Derivations
 
 
@@ -687,74 +703,47 @@ class BrefZipFuzzy(Refcat):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
 
 
-class OpenLibraryWorks(Refcat):
+class OpenLibraryEditionsByWork(Refcat):
     """
-    Extract just the works.
+    Have editions keyed by work id, 9m5.037s.
     """
     def requires(self):
-        return OpenLibraryDump()
+        return OpenLibraryEditions()
 
     def run(self):
         output = shellout("""
                           zstdcat -T0 {input} |
-                          parallel -j {n} --block 10M --pipe "jq -rc 'select(.type == \\"work\\")'" |
+                          cut -f 5 |
+                          skate-map -skip-on-empty 1 -m ff -x 'works.0.key' |
+                          LC_ALL=C sort -T {tmpdir} -S25% -k1,1 --parallel 4 |
                           zstd -T0 -c > {output}
                           """,
-                          n=self.n,
                           tmpdir=self.tmpdir,
                           input=self.input().path)
         luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
-        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
 
 
-class OpenLibraryMapped(Refcat):
+class OpenLibraryWorksSorted(Refcat):
     """
-    Map OL data with e.g. a title normalizing mapper. 4m8.341s.
+    Sorted by work id.
     """
-    mapper = luigi.Parameter(default="ts", description="mapper short name")
-
     def requires(self):
         return OpenLibraryWorks()
 
     def run(self):
         output = shellout("""
                           zstdcat -T0 {input} |
-                          skate-conv -f ol |
-                          skate-map -m {mapper} -skip-on-empty 1 |
-                          LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 |
+                          cut -f 2,5 |
+                          LC_ALL=C sort -T {tmpdir} -S25% -k1,1 --parallel 4 |
                           zstd -T0 -c > {output}
                           """,
-                          n=self.n,
-                          mapper=self.mapper,
                           tmpdir=self.tmpdir,
                           input=self.input().path)
-        luigi.LocalTarget(output).move(self.output().path)
 
-    def output(self):
-        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
-
-class BrefZipFuzzyOpenLibrary(Refcat):
-    """
-    Fuzzy matching from refs to open library (both converted to release schema).
-    """
-    mapper = luigi.Parameter(default="ts", description="mapper short name")
-
-    def requires(self):
-        return {
-            "refs": RefsMapped(mapper=self.mapper),
-            "ol": OpenLibraryMapped(mapper=self.mapper),
-        }
-
-    def run(self):
-        output = shellout(r"""
-                          skate-reduce -m ol -F <(zstdcat -T0 {refs}) -O <(zstdcat -T0 {ol}) |
-                          zstd -c -T0 > {output}
-                          """,
-                          refs=self.input().get("refs").path,
-                          ol=self.input().get("ol").path)
         luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
-        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)