From 17ddb998c3ad083077ab5b277af8d224af186c22 Mon Sep 17 00:00:00 2001
From: Martin Czygan <martin.czygan@gmail.com>
Date: Tue, 23 Mar 2021 00:41:21 +0100
Subject: add MAGDOI

---
 python/conf/settings.ini |  5 +++++
 python/refcat/tasks.py   | 29 +++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

(limited to 'python')

diff --git a/python/conf/settings.ini b/python/conf/settings.ini
index 4996374..e79153c 100644
--- a/python/conf/settings.ini
+++ b/python/conf/settings.ini
@@ -4,6 +4,9 @@
 BASE = "/bigger/.cache"
 TMPDIR = "/bigger/tmp"
 
+# Raw input file locations
+# ------------------------
+#
 # The raw input containing a single reference per line and sha1 of compressed
 # file.
 #
@@ -34,3 +37,5 @@ REFS_FILE = "/bigger/scholar/fatcat_scholar_work_fulltext.refs.json.zst"
 # Release docs from database export.
 RELEASE_EXPORT_EXPANDED_FILE = "/bigger/citations/release_export_expanded.json.zst"
 
+# MAG directory.
+MAG = "/magna/data/mag-2020-06-25"
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index c6faece..4851f2b 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -170,6 +170,13 @@ class ReleaseExportExpanded(luigi.ExternalTask, Refcat):
         return luigi.LocalTarget(path=settings.RELEASE_EXPORT_EXPANDED_FILE, format=Zstd)
 
 
+class MAGPapers(luigi.ExternalTask, Refcat):
+    """
+    Microsoft Academic dump as archived, e.g. https://archive.org/details/mag-2020-06-25
+    """
+    def output(self):
+        return luigi.LocalTarget(path=os.path.join(settings.MAG, "Papers.txt.gz"), format=Zstd)
+
 # ----8< Derivations
 
 class RefsWithUnstructured(Refcat):
@@ -1383,3 +1390,25 @@ class RGSitemapFatcatSortedKeys(Refcat):
 
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
+
+# ==== MAG
+
+class MAGDOI(Refcat):
+    """
+    List of MAG DOI.
+    """
+    def requires(self):
+        return MAGPapers()
+
+    def run(self):
+        output = shellout("""
+                          unpigz -c {input} |
+                          cut -f3 |
+                          grep -v ^$ |
+                          zstd -T0 -c > {output}
+                          """,
+                          input=self.input().path)
+        luigi.LocalTarget(output).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
-- 
cgit v1.2.3