diff options
| -rw-r--r-- | fatcat_scholar/issue_db.py | 15 |
1 files changed, 13 insertions, 2 deletions
diff --git a/fatcat_scholar/issue_db.py b/fatcat_scholar/issue_db.py index e01c6d9..3c55b51 100644 --- a/fatcat_scholar/issue_db.py +++ b/fatcat_scholar/issue_db.py @@ -124,7 +124,7 @@ def es_issue_count( .filter("term", issue=issue) .extra(request_cache=True) ) - search = search.params(request_cache="true") + search = search.params() return search.count() @@ -173,6 +173,7 @@ class IssueDB: def insert_sim_pub(self, pub: SimPubRow, cur: Any = None) -> None: if not cur: cur = self.db.cursor() + # print(pub.tuple(), file=sys.stderr) cur.execute( "INSERT OR REPLACE INTO sim_pub VALUES (?,?,?,?,?,?,?,?,?)", pub.tuple() ) @@ -258,7 +259,7 @@ class IssueDB: meta = obj["metadata"] assert "periodicals" in meta["collection"] container: Optional[fatcat_openapi_client.ContainerEntity] = None - if meta.get("issn"): + if meta.get("issn") and len(meta["issn"]) == 9: try: container = api.lookup_container(issnl=meta["issn"]) except fatcat_openapi_client.ApiException as ae: @@ -275,6 +276,8 @@ class IssueDB: container_ident=container and container.ident, wikidata_qid=container and container.wikidata_qid, ) + if isinstance(row.publisher, list): + row.publisher = row.publisher[0] self.insert_sim_pub(row, cur) cur.close() self.db.commit() @@ -350,8 +353,16 @@ class IssueDB: "SELECT sim_pubid, container_ident FROM sim_pub WHERE container_ident IS NOT NULL;" ) ) + print( + f"Loading fatcat container counts for {len(all_pub_containers)} entities...", + file=sys.stderr, + ) cur: Any = self.db.cursor() + count = 0 for (sim_pubid, container_ident) in all_pub_containers: + count += 1 + if count % 500 == 0: + print(f" {count}...", file=sys.stderr) aggs = es_container_aggs(es_client, container_ident) for agg in aggs: row = ReleaseCountsRow( |