add openalex directory source

Always run as day-specific ("TODAY") commands.

Add timeouts so command actually completes reasonably.

2 files changed, 70 insertions, 0 deletions

diff --git a/chocula/directories/__init__.py b/chocula/directories/__init__.py
index ed306c0..0bd13f8 100644
--- a/chocula/directories/__init__.py
+++ b/chocula/directories/__init__.py
@@ -19,6 +19,7 @@ from chocula.directories.issn_meta import IssnMetaLoader
 from chocula.directories.australian_era import AustralianEraLoader
 from chocula.directories.awol import AwolLoader
 from chocula.directories.mag import MagLoader
+from chocula.directories.openalex import OpenAlexLoader
 
 # sort order roughly results in metadata prioritization
 ALL_CHOCULA_DIR_CLASSES = [
@@ -43,4 +44,5 @@ ALL_CHOCULA_DIR_CLASSES = [
     SimLoader,
     ZdbFizeLoader,
     MagLoader,
+    OpenAlexLoader,
 ]
diff --git a/chocula/directories/openalex.py b/chocula/directories/openalex.py
new file mode 100644
index 0000000..478c814
--- /dev/null
+++ b/chocula/directories/openalex.py
@@ -0,0 +1,68 @@
+from typing import Iterable, Optional
+import csv
+
+from chocula.util import clean_str, clean_issn
+from chocula.common import DirectoryLoader
+from chocula.database import DirectoryInfo, HomepageUrl
+
+
+class OpenAlexLoader(DirectoryLoader):
+    """
+    TSV Columns (from schema docs):
+
+        1	JournalId	long	PRIMARY KEY
+        2	Rank	uint	(DEPRECATED)
+        3	NormalizedName	string
+        4	DisplayName	string
+        5	Issn	string (ISSN-L)
+        6	Issns	JSON list
+        7       IsOa    bool
+        8       IsInDoaj    bool
+        9	Publisher	string
+        10	Webpage	string
+        11	PaperCount	long
+        12	PaperFamilyCount	long	(DEPRECATED)
+        13	CitationCount	long
+        14	CreatedDate	DateTime
+        15	UpdatedDate	DateTime
+
+    """
+
+    source_slug = "openalex"
+
+    def open_file(self) -> Iterable:
+        return csv.DictReader(
+            open(self.config.openalex.filepath, "r"),
+            delimiter="\t",
+            fieldnames=[
+                "JournalId",
+                "Rank",
+                "NormalizedName",
+                "DisplayName",
+                "Issn",
+                "Issns",
+                "IsOa",
+                "IsInDoaj",
+                "Publisher",
+                "Webpage",
+                "PaperCount",
+                "PaperFamilyCount",
+                "CitationCount",
+                "CreatedDate",
+                "UpdatedDate",
+            ],
+        )
+
+    def parse_record(self, record) -> Optional[DirectoryInfo]:
+        info = DirectoryInfo(
+            directory_slug=self.source_slug,
+            issnl=clean_issn(record["Issn"]),
+            custom_id=record["JournalId"],
+            name=clean_str(record["DisplayName"]),
+            publisher=clean_str(record["Publisher"]),
+        )
+        homepage = HomepageUrl.from_url(record["Webpage"] or "")
+        if homepage:
+            info.homepage_urls.append(homepage)
+
+        return info