diff options
Diffstat (limited to 'python/fatcat_tools/transforms/elasticsearch.py')
| -rw-r--r-- | python/fatcat_tools/transforms/elasticsearch.py | 668 |
1 files changed, 354 insertions, 314 deletions
diff --git a/python/fatcat_tools/transforms/elasticsearch.py b/python/fatcat_tools/transforms/elasticsearch.py index 1826d4eb..e39e9ea4 100644 --- a/python/fatcat_tools/transforms/elasticsearch.py +++ b/python/fatcat_tools/transforms/elasticsearch.py @@ -1,4 +1,3 @@ - import datetime from typing import Any, Dict, Optional @@ -13,13 +12,14 @@ from fatcat_openapi_client import ( def check_kbart(year: int, archive: dict) -> Optional[bool]: - if not archive or not archive.get('year_spans'): + if not archive or not archive.get("year_spans"): return None - for span in archive['year_spans']: + for span in archive["year_spans"]: if year >= span[0] and year <= span[1]: return True return False + def test_check_kbart() -> None: assert check_kbart(1990, dict()) is None @@ -40,87 +40,89 @@ def release_to_elasticsearch(entity: ReleaseEntity, force_bool: bool = True) -> Raises exception on error (never returns None) """ - if entity.state in ('redirect', 'deleted'): + if entity.state in ("redirect", "deleted"): return dict( - ident = entity.ident, - state = entity.state, + ident=entity.ident, + state=entity.state, ) - elif entity.state != 'active': + elif entity.state != "active": raise ValueError("Unhandled entity state: {}".format(entity.state)) # First, the easy ones (direct copy) release = entity t: Dict[str, Any] = dict( - doc_index_ts=datetime.datetime.utcnow().isoformat()+"Z", - ident = release.ident, - state = release.state, - revision = release.revision, - work_id = release.work_id, - title = release.title, - subtitle = release.subtitle, - original_title = release.original_title, - release_type = release.release_type, - release_stage = release.release_stage, - withdrawn_status = release.withdrawn_status, - language = release.language, - volume = release.volume, - issue = release.issue, - pages = release.pages, - number = release.number, - license = release.license_slug, - version = release.version, - doi = release.ext_ids.doi, - pmid = release.ext_ids.pmid, - pmcid = release.ext_ids.pmcid, - isbn13 = release.ext_ids.isbn13, - wikidata_qid = release.ext_ids.wikidata_qid, - core_id = release.ext_ids.core, - arxiv_id = release.ext_ids.arxiv, - jstor_id = release.ext_ids.jstor, - ark_id = release.ext_ids.ark, - mag_id = release.ext_ids.mag, - dblp_id = release.ext_ids.dblp, - doaj_id = release.ext_ids.doaj, - hdl = release.ext_ids.hdl, - tags = [], + doc_index_ts=datetime.datetime.utcnow().isoformat() + "Z", + ident=release.ident, + state=release.state, + revision=release.revision, + work_id=release.work_id, + title=release.title, + subtitle=release.subtitle, + original_title=release.original_title, + release_type=release.release_type, + release_stage=release.release_stage, + withdrawn_status=release.withdrawn_status, + language=release.language, + volume=release.volume, + issue=release.issue, + pages=release.pages, + number=release.number, + license=release.license_slug, + version=release.version, + doi=release.ext_ids.doi, + pmid=release.ext_ids.pmid, + pmcid=release.ext_ids.pmcid, + isbn13=release.ext_ids.isbn13, + wikidata_qid=release.ext_ids.wikidata_qid, + core_id=release.ext_ids.core, + arxiv_id=release.ext_ids.arxiv, + jstor_id=release.ext_ids.jstor, + ark_id=release.ext_ids.ark, + mag_id=release.ext_ids.mag, + dblp_id=release.ext_ids.dblp, + doaj_id=release.ext_ids.doaj, + hdl=release.ext_ids.hdl, + tags=[], ) - t.update(dict( - is_oa = None, - is_longtail_oa = None, - is_preserved = None, - in_web = False, - in_dweb = False, - in_ia = False, - in_ia_sim = False, - in_kbart = None, - in_jstor = False, - in_doaj= bool(release.ext_ids.doaj), - in_shadows = False, - )) + t.update( + dict( + is_oa=None, + is_longtail_oa=None, + is_preserved=None, + in_web=False, + in_dweb=False, + in_ia=False, + in_ia_sim=False, + in_kbart=None, + in_jstor=False, + in_doaj=bool(release.ext_ids.doaj), + in_shadows=False, + ) + ) release_year = release.release_year if release.release_date: # .isoformat() results in, eg, '2010-10-22' (YYYY-MM-DD) - t['release_date'] = release.release_date.isoformat() + t["release_date"] = release.release_date.isoformat() if not release_year: release_year = release.release_date.year if release_year: - t['release_year'] = release_year + t["release_year"] = release_year - t['any_abstract'] = len(release.abstracts or []) > 0 - t['ref_count'] = len(release.refs or []) + t["any_abstract"] = len(release.abstracts or []) > 0 + t["ref_count"] = len(release.refs or []) ref_release_ids = [] - for r in (release.refs or []): + for r in release.refs or []: if r.target_release_id: ref_release_ids.append(r.target_release_id) - t['ref_release_ids'] = ref_release_ids - t['ref_linked_count'] = len(ref_release_ids) - t['contrib_count'] = len(release.contribs or []) + t["ref_release_ids"] = ref_release_ids + t["ref_linked_count"] = len(ref_release_ids) + t["contrib_count"] = len(release.contribs or []) contrib_names = [] contrib_affiliations = [] creator_ids = [] - for c in (release.contribs or []): + for c in release.contribs or []: if c.creator and c.creator.display_name: contrib_names.append(c.creator.display_name) elif c.raw_name: @@ -132,193 +134,218 @@ def release_to_elasticsearch(entity: ReleaseEntity, force_bool: bool = True) -> creator_ids.append(c.creator_id) if c.raw_affiliation: contrib_affiliations.append(c.raw_affiliation) - t['contrib_names'] = contrib_names - t['creator_ids'] = creator_ids - t['affiliations'] = contrib_affiliations + t["contrib_names"] = contrib_names + t["creator_ids"] = creator_ids + t["affiliations"] = contrib_affiliations # TODO: mapping... probably by lookup? - t['affiliation_rors'] = None + t["affiliation_rors"] = None if release.container: t.update(_rte_container_helper(release.container, release_year)) # fall back to release-level container metadata if container not linked or # missing context - if not t.get('publisher'): - t['publisher'] = release.publisher - if not t.get('container_name') and release.extra: - t['container_name'] = release.extra.get('container_name') + if not t.get("publisher"): + t["publisher"] = release.publisher + if not t.get("container_name") and release.extra: + t["container_name"] = release.extra.get("container_name") - if release.ext_ids.jstor or (release.ext_ids.doi and release.ext_ids.doi.startswith('10.2307/')): - t['in_jstor'] = True + if release.ext_ids.jstor or ( + release.ext_ids.doi and release.ext_ids.doi.startswith("10.2307/") + ): + t["in_jstor"] = True # transform file/fileset/webcapture related fields t.update(_rte_content_helper(release)) if release.ext_ids.doaj: - t['is_oa'] = True + t["is_oa"] = True if release.license_slug: # TODO: more/better checks here, particularly strict *not* OA licenses if release.license_slug.startswith("CC-"): - t['is_oa'] = True + t["is_oa"] = True if release.license_slug.startswith("ARXIV-"): - t['is_oa'] = True + t["is_oa"] = True - t['is_work_alias'] = None + t["is_work_alias"] = None extra = release.extra or dict() if extra: - if extra.get('is_oa'): + if extra.get("is_oa"): # NOTE: not actually setting this anywhere... but could - t['is_oa'] = True - if extra.get('is_work_alias') is not None: - t['is_work_alias'] = bool(extra.get('is_work_alias')) - if extra.get('longtail_oa'): + t["is_oa"] = True + if extra.get("is_work_alias") is not None: + t["is_work_alias"] = bool(extra.get("is_work_alias")) + if extra.get("longtail_oa"): # sometimes set by GROBID/matcher - t['is_oa'] = True - t['is_longtail_oa'] = True - if not t.get('container_name'): - t['container_name'] = extra.get('container_name') - if extra.get('crossref'): - if extra['crossref'].get('archive'): + t["is_oa"] = True + t["is_longtail_oa"] = True + if not t.get("container_name"): + t["container_name"] = extra.get("container_name") + if extra.get("crossref"): + if extra["crossref"].get("archive"): # all crossref archives are KBART, I believe - t['in_kbart'] = True + t["in_kbart"] = True # backwards compatible subtitle fetching - if not t['subtitle'] and extra.get('subtitle'): - if type(extra['subtitle']) == list: - t['subtitle'] = extra['subtitle'][0] + if not t["subtitle"] and extra.get("subtitle"): + if type(extra["subtitle"]) == list: + t["subtitle"] = extra["subtitle"][0] else: - t['subtitle'] = extra['subtitle'] + t["subtitle"] = extra["subtitle"] - t['first_page'] = None + t["first_page"] = None if release.pages: - first = release.pages.split('-')[0] - first = first.replace('p', '') + first = release.pages.split("-")[0] + first = first.replace("p", "") if first.isdigit(): - t['first_page'] = first + t["first_page"] = first # TODO: non-numerical first pages - t['ia_microfilm_url'] = None - if t['in_ia_sim']: + t["ia_microfilm_url"] = None + if t["in_ia_sim"]: # TODO: determine URL somehow? I think this is in flux. Will probably # need extra metadata in the container extra field. # special case as a demo for now. - if release.container_id == "hl5g6d5msjcl7hlbyyvcsbhc2u" \ - and release.release_year in (2011, 2013) \ - and release.issue \ - and release.issue.isdigit() \ - and t['first_page']: - t['ia_microfilm_url'] = "https://archive.org/details/sim_bjog_{}-{:02d}/page/n{}".format( + if ( + release.container_id == "hl5g6d5msjcl7hlbyyvcsbhc2u" + and release.release_year in (2011, 2013) + and release.issue + and release.issue.isdigit() + and t["first_page"] + ): + t[ + "ia_microfilm_url" + ] = "https://archive.org/details/sim_bjog_{}-{:02d}/page/n{}".format( release.release_year, int(release.issue) - 1, - t['first_page'], + t["first_page"], ) - t['doi_registrar'] = None - if extra and t['doi']: - for k in ('crossref', 'datacite', 'jalc'): + t["doi_registrar"] = None + if extra and t["doi"]: + for k in ("crossref", "datacite", "jalc"): if k in extra: - t['doi_registrar'] = k - if 'doi_registrar' not in t: - t['doi_registrar'] = 'crossref' + t["doi_registrar"] = k + if "doi_registrar" not in t: + t["doi_registrar"] = "crossref" - if t['doi']: - t['doi_prefix'] = t['doi'].split('/')[0] + if t["doi"]: + t["doi_prefix"] = t["doi"].split("/")[0] - if t['is_longtail_oa']: - t['is_oa'] = True + if t["is_longtail_oa"]: + t["is_oa"] = True # optionally coerce all flags from Optional[bool] to bool if force_bool: - for k in ('is_oa', 'is_longtail_oa', 'in_kbart', 'in_ia_sim', - 'in_jstor', 'in_web', 'in_dweb', 'in_shadows', - 'is_work_alias'): + for k in ( + "is_oa", + "is_longtail_oa", + "in_kbart", + "in_ia_sim", + "in_jstor", + "in_web", + "in_dweb", + "in_shadows", + "is_work_alias", + ): t[k] = bool(t[k]) - t['in_ia'] = bool(t['in_ia']) - t['is_preserved'] = bool( - t['is_preserved'] - or t['in_ia'] - or t['in_kbart'] - or t['in_jstor'] - or t.get('pmcid') - or t.get('arxiv_id') + t["in_ia"] = bool(t["in_ia"]) + t["is_preserved"] = bool( + t["is_preserved"] + or t["in_ia"] + or t["in_kbart"] + or t["in_jstor"] + or t.get("pmcid") + or t.get("arxiv_id") ) - if t['in_ia']: - t['preservation'] = 'bright' - elif t['is_preserved']: - t['preservation'] = 'dark' - elif t['in_shadows']: - t['preservation'] = 'shadows_only' + if t["in_ia"]: + t["preservation"] = "bright" + elif t["is_preserved"]: + t["preservation"] = "dark" + elif t["in_shadows"]: + t["preservation"] = "shadows_only" else: - t['preservation'] = 'none' + t["preservation"] = "none" return t + def _rte_container_helper(container: ContainerEntity, release_year: Optional[int]) -> dict: """ Container metadata sub-section of release_to_elasticsearch() """ this_year = datetime.date.today().year t = dict() - t['publisher'] = container.publisher - t['container_name'] = container.name + t["publisher"] = container.publisher + t["container_name"] = container.name # this is container.ident, not release.container_id, because there may # be a redirect involved - t['container_id'] = container.ident - t['container_issnl'] = container.issnl + t["container_id"] = container.ident + t["container_issnl"] = container.issnl issns = [container.issnl, container.issne, container.issnp] issns = list(set([i for i in issns if i])) - t['container_issns'] = issns - t['container_type'] = container.container_type - t['container_publication_status'] = container.publication_status + t["container_issns"] = issns + t["container_type"] = container.container_type + t["container_publication_status"] = container.publication_status if container.extra: c_extra = container.extra - if c_extra.get('kbart') and release_year: - if check_kbart(release_year, c_extra['kbart'].get('jstor')): - t['in_jstor'] = True - if t.get('in_kbart') or t.get('in_jstor'): - t['in_kbart'] = True - for archive in ('portico', 'lockss', 'clockss', 'pkp_pln', - 'hathitrust', 'scholarsportal', 'cariniana'): - t['in_kbart'] = t.get('in_kbart') or check_kbart(release_year, c_extra['kbart'].get(archive)) + if c_extra.get("kbart") and release_year: + if check_kbart(release_year, c_extra["kbart"].get("jstor")): + t["in_jstor"] = True + if t.get("in_kbart") or t.get("in_jstor"): + t["in_kbart"] = True + for archive in ( + "portico", + "lockss", + "clockss", + "pkp_pln", + "hathitrust", + "scholarsportal", + "cariniana", + ): + t["in_kbart"] = t.get("in_kbart") or check_kbart( + release_year, c_extra["kbart"].get(archive) + ) # recent KBART coverage is often not updated for the # current year. So for current-year publications, consider # coverage from *last* year to also be included in the # Keeper - if not t.get('in_kbart') and release_year == this_year: - t['in_kbart'] = check_kbart(this_year - 1, c_extra['kbart'].get(archive)) - - if c_extra.get('ia'): - if c_extra['ia'].get('sim') and release_year: - t['in_ia_sim'] = check_kbart(release_year, c_extra['ia']['sim']) - if c_extra['ia'].get('longtail_oa'): - t['is_longtail_oa'] = True - if c_extra.get('sherpa_romeo'): - if c_extra['sherpa_romeo'].get('color') == 'white': - t['is_oa'] = False - if c_extra.get('default_license') and c_extra.get('default_license').startswith('CC-'): - t['is_oa'] = True - if c_extra.get('doaj'): - if c_extra['doaj'].get('as_of'): - t['is_oa'] = True - t['in_doaj'] = True - if c_extra.get('road'): - if c_extra['road'].get('as_of'): - t['is_oa'] = True - if c_extra.get('szczepanski'): - if c_extra['szczepanski'].get('as_of'): - t['is_oa'] = True - if c_extra.get('country'): - t['country_code'] = c_extra['country'] - t['country_code_upper'] = c_extra['country'].upper() - if c_extra.get('publisher_type'): - t['publisher_type'] = c_extra['publisher_type'] - if c_extra.get('discipline'): - t['discipline'] = c_extra['discipline'] + if not t.get("in_kbart") and release_year == this_year: + t["in_kbart"] = check_kbart(this_year - 1, c_extra["kbart"].get(archive)) + + if c_extra.get("ia"): + if c_extra["ia"].get("sim") and release_year: + t["in_ia_sim"] = check_kbart(release_year, c_extra["ia"]["sim"]) + if c_extra["ia"].get("longtail_oa"): + t["is_longtail_oa"] = True + if c_extra.get("sherpa_romeo"): + if c_extra["sherpa_romeo"].get("color") == "white": + t["is_oa"] = False + if c_extra.get("default_license") and c_extra.get("default_license").startswith("CC-"): + t["is_oa"] = True + if c_extra.get("doaj"): + if c_extra["doaj"].get("as_of"): + t["is_oa"] = True + t["in_doaj"] = True + if c_extra.get("road"): + if c_extra["road"].get("as_of"): + t["is_oa"] = True + if c_extra.get("szczepanski"): + if c_extra["szczepanski"].get("as_of"): + t["is_oa"] = True + if c_extra.get("country"): + t["country_code"] = c_extra["country"] + t["country_code_upper"] = c_extra["country"].upper() + if c_extra.get("publisher_type"): + t["publisher_type"] = c_extra["publisher_type"] + if c_extra.get("discipline"): + t["discipline"] = c_extra["discipline"] return t + def _rte_content_helper(release: ReleaseEntity) -> dict: """ File/FileSet/WebCapture sub-section of release_to_elasticsearch() @@ -329,9 +356,9 @@ def _rte_content_helper(release: ReleaseEntity) -> dict: - any other URL """ t = dict( - file_count = len(release.files or []), - fileset_count = len(release.filesets or []), - webcapture_count = len(release.webcaptures or []), + file_count=len(release.files or []), + fileset_count=len(release.filesets or []), + webcapture_count=len(release.webcaptures or []), ) any_pdf_url = None @@ -340,38 +367,42 @@ def _rte_content_helper(release: ReleaseEntity) -> dict: ia_pdf_url = None for f in release.files or []: - if f.extra and f.extra.get('shadows'): - t['in_shadows'] = True - is_pdf = 'pdf' in (f.mimetype or '') - for release_url in (f.urls or []): + if f.extra and f.extra.get("shadows"): + t["in_shadows"] = True + is_pdf = "pdf" in (f.mimetype or "") + for release_url in f.urls or []: # first generic flags t.update(_rte_url_helper(release_url)) # then PDF specific stuff (for generating "best URL" fields) - if not f.mimetype and 'pdf' in release_url.url.lower(): + if not f.mimetype and "pdf" in release_url.url.lower(): is_pdf = True if is_pdf: any_pdf_url = release_url.url - if release_url.rel in ('webarchive', 'repository', 'repo'): + if release_url.rel in ("webarchive", "repository", "repo"): good_pdf_url = release_url.url - if '//web.archive.org/' in release_url.url or '//archive.org/' in release_url.url: + if ( + "//web.archive.org/" in release_url.url + or "//archive.org/" in release_url.url + ): best_pdf_url = release_url.url ia_pdf_url = release_url.url # here is where we bake-in PDF url priority; IA-specific - t['best_pdf_url'] = best_pdf_url or good_pdf_url or any_pdf_url - t['ia_pdf_url'] = ia_pdf_url + t["best_pdf_url"] = best_pdf_url or good_pdf_url or any_pdf_url + t["ia_pdf_url"] = ia_pdf_url for fs in release.filesets or []: - for url_obj in (fs.urls or []): + for url_obj in fs.urls or []: t.update(_rte_url_helper(url_obj)) for wc in release.webcaptures or []: - for url_obj in (wc.archive_urls or []): + for url_obj in wc.archive_urls or []: t.update(_rte_url_helper(url_obj)) return t + def _rte_url_helper(url_obj) -> dict: """ Takes a location URL ('url' and 'rel' keys) and returns generic preservation status. @@ -382,17 +413,17 @@ def _rte_url_helper(url_obj) -> dict: these will be iteratively update() into the overal object. """ t = dict() - if url_obj.rel in ('webarchive', 'repository', 'archive', 'repo'): - t['is_preserved'] = True - if '//web.archive.org/' in url_obj.url or '//archive.org/' in url_obj.url: - t['in_ia'] = True - if url_obj.url.lower().startswith('http') or url_obj.url.lower().startswith('ftp'): - t['in_web'] = True - if url_obj.rel in ('dweb', 'p2p', 'ipfs', 'dat', 'torrent'): + if url_obj.rel in ("webarchive", "repository", "archive", "repo"): + t["is_preserved"] = True + if "//web.archive.org/" in url_obj.url or "//archive.org/" in url_obj.url: + t["in_ia"] = True + if url_obj.url.lower().startswith("http") or url_obj.url.lower().startswith("ftp"): + t["in_web"] = True + if url_obj.rel in ("dweb", "p2p", "ipfs", "dat", "torrent"): # not sure what rel will be for this stuff - t['in_dweb'] = True - if '//www.jstor.org/' in url_obj.url: - t['in_jstor'] = True + t["in_dweb"] = True + if "//www.jstor.org/" in url_obj.url: + t["in_jstor"] = True return t @@ -404,50 +435,59 @@ def container_to_elasticsearch(entity, force_bool=True, stats=None): Raises exception on error (never returns None) """ - if entity.state in ('redirect', 'deleted'): + if entity.state in ("redirect", "deleted"): return dict( - ident = entity.ident, - state = entity.state, + ident=entity.ident, + state=entity.state, ) - elif entity.state != 'active': + elif entity.state != "active": raise ValueError("Unhandled entity state: {}".format(entity.state)) # First, the easy ones (direct copy) t = dict( - doc_index_ts=datetime.datetime.utcnow().isoformat()+"Z", - ident = entity.ident, - state = entity.state, - revision = entity.revision, - - name = entity.name, - publisher = entity.publisher, - container_type = entity.container_type, - publication_status= entity.publication_status, - issnl = entity.issnl, - issne = entity.issne, - issnp = entity.issnp, - wikidata_qid = entity.wikidata_qid, + doc_index_ts=datetime.datetime.utcnow().isoformat() + "Z", + ident=entity.ident, + state=entity.state, + revision=entity.revision, + name=entity.name, + publisher=entity.publisher, + container_type=entity.container_type, + publication_status=entity.publication_status, + issnl=entity.issnl, + issne=entity.issne, + issnp=entity.issnp, + wikidata_qid=entity.wikidata_qid, ) if not entity.extra: entity.extra = dict() - for key in ('country', 'languages', 'mimetypes', 'original_name', - 'first_year', 'last_year', 'aliases', 'abbrev', 'region', - 'discipline', 'publisher_type'): + for key in ( + "country", + "languages", + "mimetypes", + "original_name", + "first_year", + "last_year", + "aliases", + "abbrev", + "region", + "discipline", + "publisher_type", + ): if entity.extra.get(key): t[key] = entity.extra[key] - if entity.extra.get('dblp') and entity.extra['dblp'].get('prefix'): - t['dblp_prefix'] = entity.extra['dblp']['prefix'] + if entity.extra.get("dblp") and entity.extra["dblp"].get("prefix"): + t["dblp_prefix"] = entity.extra["dblp"]["prefix"] - if 'country' in t: - t['country_code'] = t.pop('country') + if "country" in t: + t["country_code"] = t.pop("country") - t['issns'] = [entity.issnl, entity.issne, entity.issnp] - for key in ('issnp', 'issne'): + t["issns"] = [entity.issnl, entity.issne, entity.issnp] + for key in ("issnp", "issne"): if entity.extra.get(key): - t['issns'].append(entity.extra[key]) - t['issns'] = list(set([i for i in t['issns'] if i])) + t["issns"].append(entity.extra[key]) + t["issns"] = list(set([i for i in t["issns"] if i])) in_doaj = None in_road = None @@ -459,72 +499,72 @@ def container_to_elasticsearch(entity, force_bool=True, stats=None): keepers = [] extra = entity.extra - if extra.get('doaj'): - if extra['doaj'].get('as_of'): + if extra.get("doaj"): + if extra["doaj"].get("as_of"): in_doaj = True - if extra.get('road'): - if extra['road'].get('as_of'): + if extra.get("road"): + if extra["road"].get("as_of"): in_road = True - if extra.get('szczepanski'): - if extra['szczepanski'].get('as_of'): + if extra.get("szczepanski"): + if extra["szczepanski"].get("as_of"): is_oa = True - if extra.get('default_license'): - if extra['default_license'].startswith('CC-'): + if extra.get("default_license"): + if extra["default_license"].startswith("CC-"): is_oa = True - t['sherpa_romeo_color'] = None - if extra.get('sherpa_romeo'): - t['sherpa_romeo_color'] = extra['sherpa_romeo'].get('color') - if extra['sherpa_romeo'].get('color') == 'white': + t["sherpa_romeo_color"] = None + if extra.get("sherpa_romeo"): + t["sherpa_romeo_color"] = extra["sherpa_romeo"].get("color") + if extra["sherpa_romeo"].get("color") == "white": is_oa = False - if extra.get('kbart'): + if extra.get("kbart"): any_kbart = True - if extra['kbart'].get('jstor'): + if extra["kbart"].get("jstor"): any_jstor = True - for k, v in extra['kbart'].items(): + for k, v in extra["kbart"].items(): if v and isinstance(v, dict): keepers.append(k) - if extra.get('ia'): - if extra['ia'].get('sim'): + if extra.get("ia"): + if extra["ia"].get("sim"): any_ia_sim = True - if extra['ia'].get('longtail_oa'): + if extra["ia"].get("longtail_oa"): is_longtail_oa = True - t['is_superceded'] = bool(extra.get('superceded')) + t["is_superceded"] = bool(extra.get("superceded")) - t['keepers'] = keepers - t['in_doaj'] = bool(in_doaj) - t['in_road'] = bool(in_road) - t['any_kbart'] = bool(any_kbart) + t["keepers"] = keepers + t["in_doaj"] = bool(in_doaj) + t["in_road"] = bool(in_road) + t["any_kbart"] = bool(any_kbart) if force_bool: - t['is_oa'] = bool(in_doaj or in_road or is_oa) - t['is_longtail_oa'] = bool(is_longtail_oa) - t['any_jstor'] = bool(any_jstor) - t['any_ia_sim'] = bool(any_ia_sim) + t["is_oa"] = bool(in_doaj or in_road or is_oa) + t["is_longtail_oa"] = bool(is_longtail_oa) + t["any_jstor"] = bool(any_jstor) + t["any_ia_sim"] = bool(any_ia_sim) else: - t['is_oa'] = in_doaj or in_road or is_oa - t['is_longtail_oa'] = is_longtail_oa - t['any_jstor'] = any_jstor - t['any_ia_sim'] = any_ia_sim + t["is_oa"] = in_doaj or in_road or is_oa + t["is_longtail_oa"] = is_longtail_oa + t["any_jstor"] = any_jstor + t["any_ia_sim"] = any_ia_sim # mix in stats, if provided if stats: - t['releases_total'] = stats['total'] - t['preservation_bright'] = stats['preservation']['bright'] - t['preservation_dark'] = stats['preservation']['dark'] - t['preservation_shadows_only'] = stats['preservation']['shadows_only'] - t['preservation_none'] = stats['preservation']['none'] + t["releases_total"] = stats["total"] + t["preservation_bright"] = stats["preservation"]["bright"] + t["preservation_dark"] = stats["preservation"]["dark"] + t["preservation_shadows_only"] = stats["preservation"]["shadows_only"] + t["preservation_none"] = stats["preservation"]["none"] return t def _type_of_edit(edit: EntityEdit) -> str: if edit.revision is None and edit.redirect_ident is None: - return 'delete' + return "delete" elif edit.redirect_ident: # redirect - return 'update' + return "update" elif edit.prev_revision is None and edit.redirect_ident is None and edit.revision: - return 'create' + return "create" else: - return 'update' + return "update" def changelog_to_elasticsearch(entity: ChangelogEntry) -> Dict[str, Any]: @@ -536,7 +576,7 @@ def changelog_to_elasticsearch(entity: ChangelogEntry) -> Dict[str, Any]: editgroup = entity.editgroup t = dict( - doc_index_ts=datetime.datetime.utcnow().isoformat()+"Z", + doc_index_ts=datetime.datetime.utcnow().isoformat() + "Z", index=entity.index, editgroup_id=entity.editgroup_id, timestamp=entity.timestamp.isoformat(), @@ -547,8 +587,8 @@ def changelog_to_elasticsearch(entity: ChangelogEntry) -> Dict[str, Any]: ) extra = editgroup.extra or dict() - if extra.get('agent'): - t['agent'] = extra['agent'] + if extra.get("agent"): + t["agent"] = extra["agent"] containers = [_type_of_edit(e) for e in editgroup.edits.containers] creators = [_type_of_edit(e) for e in editgroup.edits.creators] @@ -558,27 +598,27 @@ def changelog_to_elasticsearch(entity: ChangelogEntry) -> Dict[str, Any]: releases = [_type_of_edit(e) for e in editgroup.edits.releases] works = [_type_of_edit(e) for e in editgroup.edits.works] - t['containers'] = len(containers) - t['new_containers'] = len([e for e in containers if e == 'create']) - t['creators'] = len(creators) - t['new_creators'] = len([e for e in creators if e == 'create']) - t['files'] = len(files) - t['new_files'] = len([e for e in files if e == 'create']) - t['filesets'] = len(filesets) - t['new_filesets'] = len([e for e in filesets if e == 'create']) - t['webcaptures'] = len(webcaptures) - t['new_webcaptures'] = len([e for e in webcaptures if e == 'create']) - t['releases'] = len(releases) - t['new_releases'] = len([e for e in releases if e == 'create']) - t['works'] = len(works) - t['new_works'] = len([e for e in works if e == 'create']) + t["containers"] = len(containers) + t["new_containers"] = len([e for e in containers if e == "create"]) + t["creators"] = len(creators) + t["new_creators"] = len([e for e in creators if e == "create"]) + t["files"] = len(files) + t["new_files"] = len([e for e in files if e == "create"]) + t["filesets"] = len(filesets) + t["new_filesets"] = len([e for e in filesets if e == "create"]) + t["webcaptures"] = len(webcaptures) + t["new_webcaptures"] = len([e for e in webcaptures if e == "create"]) + t["releases"] = len(releases) + t["new_releases"] = len([e for e in releases if e == "create"]) + t["works"] = len(works) + t["new_works"] = len([e for e in works if e == "create"]) all_edits = containers + creators + files + filesets + webcaptures + releases + works - t['created'] = len([e for e in all_edits if e == 'create']) - t['updated'] = len([e for e in all_edits if e == 'update']) - t['deleted'] = len([e for e in all_edits if e == 'delete']) - t['total'] = len(all_edits) + t["created"] = len([e for e in all_edits if e == "create"]) + t["updated"] = len([e for e in all_edits if e == "update"]) + t["deleted"] = len([e for e in all_edits if e == "delete"]) + t["total"] = len(all_edits) return t @@ -590,47 +630,47 @@ def file_to_elasticsearch(entity: FileEntity) -> Dict[str, Any]: Raises exception on error (never returns None) """ - if entity.state in ('redirect', 'deleted'): + if entity.state in ("redirect", "deleted"): return dict( - ident = entity.ident, - state = entity.state, + ident=entity.ident, + state=entity.state, ) - elif entity.state != 'active': + elif entity.state != "active": raise ValueError("Unhandled entity state: {}".format(entity.state)) # First, the easy ones (direct copy) t = dict( - doc_index_ts=datetime.datetime.utcnow().isoformat()+"Z", - ident = entity.ident, - state = entity.state, - revision = entity.revision, - release_ids = entity.release_ids, - release_count = len(entity.release_ids), - mimetype = entity.mimetype, - size_bytes = entity.size, - sha1 = entity.sha1, - sha256 = entity.sha256, - md5 = entity.md5, + doc_index_ts=datetime.datetime.utcnow().isoformat() + "Z", + ident=entity.ident, + state=entity.state, + revision=entity.revision, + release_ids=entity.release_ids, + release_count=len(entity.release_ids), + mimetype=entity.mimetype, + size_bytes=entity.size, + sha1=entity.sha1, + sha256=entity.sha256, + md5=entity.md5, ) parsed_urls = [tldextract.extract(u.url) for u in entity.urls] - t['hosts'] = list(set(['.'.join([seg for seg in pu if seg]) for pu in parsed_urls])) - t['domains'] = list(set([pu.registered_domain for pu in parsed_urls])) - t['rels'] = list(set([u.rel for u in entity.urls])) + t["hosts"] = list(set([".".join([seg for seg in pu if seg]) for pu in parsed_urls])) + t["domains"] = list(set([pu.registered_domain for pu in parsed_urls])) + t["rels"] = list(set([u.rel for u in entity.urls])) - t['in_ia'] = bool('archive.org' in t['domains']) - t['in_ia_petabox'] = bool('archive.org' in t['hosts']) + t["in_ia"] = bool("archive.org" in t["domains"]) + t["in_ia_petabox"] = bool("archive.org" in t["hosts"]) any_url = None good_url = None best_url = None - for release_url in (entity.urls or []): + for release_url in entity.urls or []: any_url = release_url.url - if release_url.rel in ('webarchive', 'repository'): + if release_url.rel in ("webarchive", "repository"): good_url = release_url.url - if '//web.archive.org/' in release_url.url or '//archive.org/' in release_url.url: + if "//web.archive.org/" in release_url.url or "//archive.org/" in release_url.url: best_url = release_url.url # here is where we bake-in priority; IA-specific - t['best_url'] = best_url or good_url or any_url + t["best_url"] = best_url or good_url or any_url return t |