aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorMartin Czygan <martin.czygan@gmail.com>2022-01-14 00:46:23 +0100
committerMartin Czygan <martin.czygan@gmail.com>2022-01-14 00:46:23 +0100
commit7cc9e8685361117de5766cbd99c8dd56f0c28cc6 (patch)
tree561405de406c3b71320607b7ef9fba99d687da00 /python
parenta0c5bd6fe86b73c5824ea8966f785ec298f7d188 (diff)
downloadrefcat-7cc9e8685361117de5766cbd99c8dd56f0c28cc6.tar.gz
refcat-7cc9e8685361117de5766cbd99c8dd56f0c28cc6.zip
tasks: fix Wikipedia20211201DOI
Diffstat (limited to 'python')
-rw-r--r--python/README.md5
-rw-r--r--python/refcat/tasks.py14
2 files changed, 17 insertions, 2 deletions
diff --git a/python/README.md b/python/README.md
index f449477..071286c 100644
--- a/python/README.md
+++ b/python/README.md
@@ -93,3 +93,8 @@ OpenLibraryWorks
## Dependencies
![](notes/deps.png)
+
+
+## TODO
+
+* [ ] wrap up refcat
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 071184e..d774038 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -290,6 +290,12 @@ class WikipediaCitations20211201(luigi.ExternalTask, Refcat):
"""
Update wikipedia citations dataset: https://archive.org/details/wikipedia-citations-enwiki-20211201
+ Generated with https://github.com/dissemin/wikiciteparser.
+
+ * DOI, PMID, PMCID, arxiv-id, webarchive (prefix version)
+ * other identifier exact
+ * does not have some title cases
+
Example line:
{
@@ -1574,21 +1580,25 @@ class Wikipedia20211201DOI(Refcat):
return WikipediaCitations20211201()
def run(self):
- with tempfile.NamedTemporaryFile(delete=False) as tf:
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as tf:
with self.input().open() as handle:
for line in handle:
doc = json.loads(line)
+ if not doc["page_title"]:
+ continue
for i, ref in enumerate(doc.get("refs", [])):
if not "ID_list" in ref:
continue
if not "DOI" in ref["ID_list"]:
continue
- doi = ref["ID_list"]["DOI"]
+ doi = ref["ID_list"]["DOI"].strip()
reduced = doc
reduced["refs"] = []
reduced["index"] = i
reduced["Title"] = ref.get("Title")
fields = [doi, doc["page_title"], json.dumps(reduced)]
+ if not all(fields):
+ continue
tf.write("\t".join(fields) + "\n")
output = shellout("LC_ALL=C sort -S30% {input} > {output}", input=tf.name)
luigi.LocalTarget(output).move(self.output().path)