summaryrefslogtreecommitdiffstats
path: root/python/parse_jalc_xml.py
diff options
context:
space:
mode:
Diffstat (limited to 'python/parse_jalc_xml.py')
-rw-r--r--python/parse_jalc_xml.py209
1 files changed, 0 insertions, 209 deletions
diff --git a/python/parse_jalc_xml.py b/python/parse_jalc_xml.py
deleted file mode 100644
index d7817df9..00000000
--- a/python/parse_jalc_xml.py
+++ /dev/null
@@ -1,209 +0,0 @@
-
-import sys
-import json
-import datetime
-import unicodedata
-from bs4 import BeautifulSoup
-from bs4.element import NavigableString
-
-
-DATE_FMT = "%Y-%m-%d"
-
-def is_cjk(s):
- if not s:
- return False
- return unicodedata.name(s[0]).startswith("CJK")
-
-class JalcXmlParser():
- """
- Converts JALC DOI metadata (in XML/RDF format) to fatcat release entity
-
- NOTE: some JALC DOIs seem to get cross-registered with Crossref
- """
-
- def __init__(self):
- pass
-
- def parse_file(self, handle):
-
- # 1. open with beautiful soup
- soup = BeautifulSoup(handle, "xml")
-
- # 2. iterate over articles, call parse_article on each
- for record in soup.find_all("Description"):
- resp = self.parse_record(record)
- print(json.dumps(resp))
- #sys.exit(-1)
-
-
- def parse_record(self, record):
- """
- In JALC metadata, both English and Japanese records are given for most
- fields.
- """
-
- #extra = dict()
- #extra_jalc = dict()
-
- titles = record.find_all("title")
- title = titles[0].string.strip()
- original_title = None
- if title.endswith('.'):
- title = title[:-1]
- if len(titles) > 1:
- original_title = titles[1].string.strip()
- if original_title.endswith('.'):
- original_title = original_title[:-1]
-
- doi = None
- if record.doi:
- doi = record.doi.string.lower().strip()
- assert doi.startswith('10.')
-
- contribs = []
- people = record.find_all("Person")
- if people and (len(people) % 2 == 0) and is_cjk(people[1].find('name').string):
- # both english and japanese names are included
- for i in range(int(len(people)/2)):
- # both english and japanese names are included for every author
- eng = people[i*2]
- jpn = people[i*2 + 1]
- raw_name = eng.find('name')
- orig_name = jpn.find('name')
- if not raw_name:
- raw_name = orig_name
- contrib = dict(
- raw_name=raw_name.string,
- role='author',
- )
- if raw_name and orig_name:
- contrib['extra'] = dict(original_name=orig_name.string)
- contribs.append(contrib)
- elif people:
- for eng in people:
- raw_name = eng.find('name')
- contrib = dict(
- raw_name=eng.find('name').string,
- role='author',
- )
- contribs.append(contrib)
-
- release_year = None
- release_date = None
- date = record.date or None
- if date:
- date = date.string
- if len(date) is 10:
- release_date = datetime.datetime.strptime(date['completed-date'], DATE_FMT).date()
- release_year = release_date.year
- release_date = release_date.isoformat()
- elif len(date) is 4:
- release_year = int(date)
-
- pages = None
- if record.startingPage:
- pages = record.startingPage.string
- if record.endingPage:
- pages = "{}-{}".format(pages, record.endingPage.string)
- volume = None
- if record.volume:
- volume = record.volume.string
- issue = None
- if record.number:
- # note: number/issue transform
- issue = record.number.string
-
- issn = None
- issn_list = record.find_all("issn")
- if issn_list:
- # if we wanted the other ISSNs, would also need to uniq the list.
- # But we only need one to lookup ISSN-L/container
- issn = issn_list[0].string
-
- container = dict()
- container_extra = dict()
- container_name = None
- if record.publicationName:
- pubs = [p.string.strip() for p in record.find_all("publicationName")]
- pubs = [p for p in pubs if p]
- assert(pubs)
- if len(pubs) > 1 and pubs[0] == pubs[1]:
- pubs = [pubs[0]]
- elif len(pubs) > 1 and is_cjk(pubs[0]):
- # ordering is not reliable
- pubs = [pubs[1], pubs[0]]
- container_name = pubs[0]
- container['name'] = container_name
- if len(pubs) > 1:
- orig_container_name = pubs[1]
- container_extra['original_name'] = pubs[1]
- publisher = None
- if record.publisher:
- pubs = [p.string.strip() for p in record.find_all("publisher")]
- pubs = [p for p in pubs if p]
- if len(pubs) > 1 and pubs[0] == pubs[1]:
- pubs = [pubs[0]]
- elif len(pubs) > 1 and is_cjk(pubs[0]):
- # ordering is not reliable
- pubs = [pubs[1], pubs[0]]
- publisher = pubs[0]
- container['publisher'] = publisher
- if len(pubs) > 1:
- container_extra['publisher_alt_name'] = pubs[1]
- if container_extra:
- container['extra'] = container_extra
- if not container:
- container = None
-
- # the vast majority of works are in japanese
- # TODO: any indication when *not* in japanese?
- lang = "ja"
-
- # reasonable default for this collection
- release_type = "article-journal"
-
- re = dict(
- work_id=None,
- title=title,
- original_title=original_title,
- release_type="article-journal",
- release_status='submitted', # XXX: source_type?
- release_date=release_date,
- release_year=release_year,
- #arxiv_id
- doi=doi,
- #pmid
- #pmcid
- #isbn13 # never in Article
- volume=volume,
- issue=issue,
- pages=pages,
- publisher=publisher,
- language=lang,
- #license_slug # not in MEDLINE
-
- # content, mimetype, lang
- #abstracts=abstracts,
-
- # raw_name, role, raw_affiliation, extra
- contribs=contribs,
-
- # name, type, publisher, issnl
- # extra: issnp, issne, original_name, languages, country
- container=container,
-
- # extra:
- # withdrawn_date
- # translation_of
- # subtitle
- # aliases
- # container_name
- # group-title
- # pubmed: retraction refs
- #extra=extra,
- )
- return re
-
-if __name__=='__main__':
- parser = JalcXmlParser()
- parser.parse_file(open(sys.argv[1]))