diff options
| -rw-r--r-- | fatcat_scholar/issue_db.py | 22 | ||||
| -rw-r--r-- | fatcat_scholar/transform.py | 3 | ||||
| -rw-r--r-- | fatcat_scholar/work_pipeline.py | 26 |
3 files changed, 39 insertions, 12 deletions
diff --git a/fatcat_scholar/issue_db.py b/fatcat_scholar/issue_db.py index 5278750..4f5ff53 100644 --- a/fatcat_scholar/issue_db.py +++ b/fatcat_scholar/issue_db.py @@ -25,6 +25,20 @@ class SimPubRow: def tuple(self): return (self.sim_pubid, self.pub_collection, self.title, self.issn, self.pub_type, self.publisher, self.container_issnl, self.container_ident, self.wikidata_qid) + @classmethod + def from_tuple(cls, row: Any) -> "SimPubRow": + return SimPubRow( + sim_pubid=row[0], + pub_collection=row[1], + title=row[2], + issn=row[3], + pub_type=row[4], + publisher=row[5], + container_issnl=row[6], + container_ident=row[7], + wikidata_qid=row[8], + ) + @dataclass class SimIssueRow: """ @@ -45,7 +59,7 @@ class SimIssueRow: return (self.issue_item, self.sim_pubid, self.year, self.volume, self.issue, self.first_page, self.last_page, self.release_count) @classmethod - def from_tuple(self, row: Any): + def from_tuple(cls, row: Any) -> "SimIssueRow": return SimIssueRow( issue_item=row[0], sim_pubid=row[1], @@ -166,6 +180,12 @@ class IssueDB(): return None return SimIssueRow.from_tuple(row[0]) + def lookup_pub(self, sim_pubid: str) -> Optional[SimPubRow]: + row = list(self.db.execute("SELECT * FROM sim_pub WHERE sim_pubid = ?;", [sim_pubid])) + if not row: + return None + return SimPubRow.from_tuple(row[0]) + def load_pubs(self, json_lines: Sequence[str], api: Any): """ Reads a file (or some other iterator) of JSON lines, parses them into a diff --git a/fatcat_scholar/transform.py b/fatcat_scholar/transform.py index a86fe15..d858a4c 100644 --- a/fatcat_scholar/transform.py +++ b/fatcat_scholar/transform.py @@ -184,11 +184,12 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]: tags: List[str] = [] work_ident: Optional[str] = None abstracts: List[ScholarAbstract] = [] - fulltext: Optional[ScholarFulltext] + fulltext: Optional[ScholarFulltext] = None ia_sim: Optional[ScholarSim] = None if heavy.sim_fulltext is not None: ia_sim = es_sim_from_sim(heavy.sim_fulltext) + fulltext = es_fulltext_from_sim(heavy.sim_fulltext) if heavy.doc_type == DocType.sim_page: assert ia_sim is not None diff --git a/fatcat_scholar/work_pipeline.py b/fatcat_scholar/work_pipeline.py index 081878c..ebc2923 100644 --- a/fatcat_scholar/work_pipeline.py +++ b/fatcat_scholar/work_pipeline.py @@ -11,8 +11,9 @@ import internetarchive from fatcat_scholar.api_entities import * from fatcat_scholar.djvu import djvu_extract_leaf_texts from fatcat_scholar.sandcrawler import SandcrawlerPostgrestClient, SandcrawlerMinioClient -from fatcat_scholar.issue_db import IssueDB, SimIssueRow +from fatcat_scholar.issue_db import IssueDB, SimIssueRow, SimPubRow from fatcat_scholar.schema import es_biblio_from_release, es_release_from_release, DocType, IntermediateBundle +from fatcat_scholar.sim_pipeline import truncate_pub_meta, truncate_issue_meta def parse_pages(raw: str) -> Tuple[Optional[int], Optional[int]]: @@ -124,12 +125,13 @@ class WorkPipeline(): return self.issue_db.lookup_issue(sim_pubid=sim_pubid, volume=release.volume, issue=release.issue) - def fetch_sim(self, issue_db_row: SimIssueRow, pages: str, release_ident: str) -> Optional[Any]: + def fetch_sim(self, issue_db_row: SimIssueRow, issue_db_pub_row: SimPubRow, pages: str, release_ident: str) -> Optional[Any]: """ issue_item pages: str page_texts: list - page_number + page_num + leaf_num raw_text release_ident: Optional[str] pub_item_metadata @@ -142,13 +144,14 @@ class WorkPipeline(): # fetch full metadata from API issue_meta = self.ia_client.get_metadata(issue_db_row.issue_item) - # XXX: pub_meta = self.ia_client.get_metadata(issue_db_row.pub_collection) - pub_meta = None + pub_meta = self.ia_client.get_metadata(issue_db_pub_row.pub_collection) + leaf_index = dict() leaf_list = [] assert 'page_numbers' in issue_meta for entry in issue_meta['page_numbers'].get('pages', []): page_num = entry['pageNumber'] + leaf_index[entry['leafNum']] = page_num if not (page_num and page_num.isdigit()): continue page_num = int(page_num) @@ -172,16 +175,16 @@ class WorkPipeline(): leaf_dict = djvu_extract_leaf_texts(djvu_xml, only_leaves=leaf_list) - for leaf, raw_text in leaf_dict.items(): - page_texts.append(dict(page_number=leaf, raw_text=raw_text)) + for leaf_num, raw_text in leaf_dict.items(): + page_texts.append(dict(page_num=leaf_index.get(leaf_num), leaf_num=leaf_num, raw_text=raw_text)) return dict( issue_item=issue_db_row.issue_item, pages=pages, page_texts=page_texts, release_ident=release_ident, - pub_item_metadata=pub_meta, - issue_item_metadata=issue_item.metadata, + pub_item_metadata=truncate_pub_meta(pub_meta), + issue_item_metadata=truncate_issue_meta(issue_meta), ) def process_release_list(self, releases: List[ReleaseEntity]) -> IntermediateBundle: @@ -226,8 +229,11 @@ class WorkPipeline(): #print(f"release_{release.ident}: sim_issue={sim_issue}", file=sys.stderr) if not sim_issue: continue + sim_pub = self.issue_db.lookup_pub(sim_issue.sim_pubid) + if not sim_pub: + continue # XXX: control flow tweak? - sim_fulltext = self.fetch_sim(sim_issue, release.pages, release.ident) + sim_fulltext = self.fetch_sim(sim_issue, sim_pub, release.pages, release.ident) if sim_fulltext: break |