diff options
author | Bryan Newbold <bnewbold@archive.org> | 2021-11-22 13:53:48 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2021-11-22 13:58:34 -0800 |
commit | e72084a274145adc2dbcc8371bf6e4b26e129349 (patch) | |
tree | 49a124fccf0089550d438c323e77937660e2806a /chocula | |
parent | 8d2f11fc0221a90732b3b13a3df1b16ea8acadeb (diff) | |
download | chocula-e72084a274145adc2dbcc8371bf6e4b26e129349.tar.gz chocula-e72084a274145adc2dbcc8371bf6e4b26e129349.zip |
add openalex directory source
Always run as day-specific ("TODAY") commands.
Add timeouts so command actually completes reasonably.
Diffstat (limited to 'chocula')
-rw-r--r-- | chocula/directories/__init__.py | 2 | ||||
-rw-r--r-- | chocula/directories/openalex.py | 68 |
2 files changed, 70 insertions, 0 deletions
diff --git a/chocula/directories/__init__.py b/chocula/directories/__init__.py index ed306c0..0bd13f8 100644 --- a/chocula/directories/__init__.py +++ b/chocula/directories/__init__.py @@ -19,6 +19,7 @@ from chocula.directories.issn_meta import IssnMetaLoader from chocula.directories.australian_era import AustralianEraLoader from chocula.directories.awol import AwolLoader from chocula.directories.mag import MagLoader +from chocula.directories.openalex import OpenAlexLoader # sort order roughly results in metadata prioritization ALL_CHOCULA_DIR_CLASSES = [ @@ -43,4 +44,5 @@ ALL_CHOCULA_DIR_CLASSES = [ SimLoader, ZdbFizeLoader, MagLoader, + OpenAlexLoader, ] diff --git a/chocula/directories/openalex.py b/chocula/directories/openalex.py new file mode 100644 index 0000000..478c814 --- /dev/null +++ b/chocula/directories/openalex.py @@ -0,0 +1,68 @@ +from typing import Iterable, Optional +import csv + +from chocula.util import clean_str, clean_issn +from chocula.common import DirectoryLoader +from chocula.database import DirectoryInfo, HomepageUrl + + +class OpenAlexLoader(DirectoryLoader): + """ + TSV Columns (from schema docs): + + 1 JournalId long PRIMARY KEY + 2 Rank uint (DEPRECATED) + 3 NormalizedName string + 4 DisplayName string + 5 Issn string (ISSN-L) + 6 Issns JSON list + 7 IsOa bool + 8 IsInDoaj bool + 9 Publisher string + 10 Webpage string + 11 PaperCount long + 12 PaperFamilyCount long (DEPRECATED) + 13 CitationCount long + 14 CreatedDate DateTime + 15 UpdatedDate DateTime + + """ + + source_slug = "openalex" + + def open_file(self) -> Iterable: + return csv.DictReader( + open(self.config.openalex.filepath, "r"), + delimiter="\t", + fieldnames=[ + "JournalId", + "Rank", + "NormalizedName", + "DisplayName", + "Issn", + "Issns", + "IsOa", + "IsInDoaj", + "Publisher", + "Webpage", + "PaperCount", + "PaperFamilyCount", + "CitationCount", + "CreatedDate", + "UpdatedDate", + ], + ) + + def parse_record(self, record) -> Optional[DirectoryInfo]: + info = DirectoryInfo( + directory_slug=self.source_slug, + issnl=clean_issn(record["Issn"]), + custom_id=record["JournalId"], + name=clean_str(record["DisplayName"]), + publisher=clean_str(record["Publisher"]), + ) + homepage = HomepageUrl.from_url(record["Webpage"] or "") + if homepage: + info.homepage_urls.append(homepage) + + return info |