add BibliorefSortedIdent, RefsSortedIdent

we'll need this to complete the unmatched refs

1 files changed, 52 insertions, 6 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 8459608..5eaca50 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -177,6 +177,7 @@ class MAGPapers(luigi.ExternalTask, Refcat):
     def output(self):
         return luigi.LocalTarget(path=os.path.join(settings.MAG, "Papers.txt.gz"), format=Zstd)
 
+
 class WikipediaCitationsMinimalDataset(luigi.ExternalTask, Refcat):
     """
     From archive.org/details/wikipedia_citations_2020-07-14 (Wikipedia
@@ -193,6 +194,7 @@ class WikipediaCitationsMinimalDataset(luigi.ExternalTask, Refcat):
 
 # ----8< Derivations
 
+
 class RefsWithUnstructured(Refcat):
     """
     Augment refs with data from unstructured. Do this first, so we can use it
@@ -1097,8 +1099,6 @@ class RefsFatcatClusters(Refcat):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
 
 
-
-
 # ==== new style zippy biblioref generation
 
 
@@ -1228,7 +1228,8 @@ class BiblioRefV2(Refcat):
                      skate-bref-id |
                      zstd -T0 >> {output}
                      """,
-                     input=target.path, output=tmpf)
+                     input=target.path,
+                     output=tmpf)
         luigi.LocalTarget(tmpf).move(self.output().path)
 
     def output(self):
@@ -1317,8 +1318,10 @@ class RGSitemapFatcatSortedKeys(Refcat):
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
 
+
 # ==== MAG
 
+
 class MAGDOI(Refcat):
     """
     List of MAG DOI.
@@ -1339,13 +1342,14 @@ class MAGDOI(Refcat):
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
 
+
 # ==== WikipediaCitations
 
+
 class BiblioRefWikiDOISortedKeys(Refcat):
     """
     Sorted DOI keys from wikipedia.
     """
-
     def requires(self):
         return WikipediaCitationsMinimalDataset()
 
@@ -1355,14 +1359,15 @@ class BiblioRefWikiDOISortedKeys(Refcat):
                           skate-wikipedia-doi |
                           LC_ALL=C sort -S 10% -k2,2 |
                           zstd -T0 -c > {output}
-                          """, input=self.input().path)
+                          """,
+                          input=self.input().path)
         luigi.LocalTarget(output).move(self.output().path)
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
 
-class BiblioRefWiki(Refcat):
 
+class BiblioRefWiki(Refcat):
     def requires(self):
         return {
             "wiki": BiblioRefWikiDOISortedKeys(),
@@ -1380,3 +1385,44 @@ class BiblioRefWiki(Refcat):
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+
+
+# ==== Prepare unmatched
+
+
+class BibliorefSortedIdent(Refcat):
+    def requires(self):
+        return BiblioRefV2()
+
+    def run(self):
+        output = shellout("""
+                          zstdcat -T0 {input} |
+                          skate-derive-key -b 50000 -verbose -F source_release_ident |
+                          LC_ALL=C sort -k2,2 -S 35% --parallel 6 --compress-program pzstd -T {tmpdir} |
+                          zstd -T0 -c > {output}
+                          """,
+                          tmpdir=self.tmpdir,
+                          input=self.input().path)
+        luigi.LocalTarget(output).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+
+
+class RefSortedIdent(Refcat):
+    def requires(self):
+        return RefsWithUnstructured()
+
+    def run(self):
+        output = shellout("""
+                          zstdcat -T0 {input} |
+                          skate-derive-key -b 50000 -verbose -F release_ident |
+                          LC_ALL=C sort -k2,2 -S 35% --parallel 6 --compress-program pzstd -T {tmpdir} |
+                          zstd -T0 -c > {output}
+                          """,
+                          tmpdir=self.tmpdir,
+                          input=self.input().path)
+        luigi.LocalTarget(output).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)