diff options
| -rw-r--r-- | python/fatcat_tools/importers/common.py | 5 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/crossref.py | 4 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/issn.py | 4 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/matched.py | 11 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/orcid.py | 4 | ||||
| -rw-r--r-- | python/fatcat_web/templates/editor_editgroups.html (renamed from python/fatcat_web/templates/editor_changelog.html) | 0 |
6 files changed, 16 insertions, 12 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index e39ec6c9..06897bee 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -96,8 +96,9 @@ class FatcatImporter: if decode_kafka: rows = [msg.value.decode('utf-8') for msg in rows] self.counts['processed_lines'] += len(rows) - eg = self._editgroup() - self.create_batch(rows, editgroup_id=eg.editgroup_id) + #eg = self._editgroup() + #self.create_batch(rows, editgroup_id=eg.editgroup_id) + self.create_batch(rows) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py index ed60a78c..6365e491 100644 --- a/python/fatcat_tools/importers/crossref.py +++ b/python/fatcat_tools/importers/crossref.py @@ -313,7 +313,7 @@ class CrossrefImporter(FatcatImporter): self.api.create_release(re, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Current work/release pairing disallows batch creation of releases. Could do batch work creation and then match against releases, but meh.""" release_batch = [] @@ -331,5 +331,5 @@ class CrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_release_batch(release_batch, autoaccept="true") self.counts['insert'] += len(release_batch) diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py index 02a1eea0..f4d525a4 100644 --- a/python/fatcat_tools/importers/issn.py +++ b/python/fatcat_tools/importers/issn.py @@ -80,10 +80,10 @@ class IssnImporter(FatcatImporter): self.api.create_container(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_issn_row(l) for l in batch if (l is not None)] objects = [o for o in objects if (o is not None)] - self.api.create_container_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_container_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py index 0b77bcf0..1e5c22f7 100644 --- a/python/fatcat_tools/importers/matched.py +++ b/python/fatcat_tools/importers/matched.py @@ -142,15 +142,18 @@ class MatchedImporter(FatcatImporter): self.api.update_file(fe.ident, fe, editgroup_id=editgroup_id) self.counts['update'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_matched_dict(json.loads(l)) for l in batch if l != None] new_objects = [o for o in objects if o != None and o.ident == None] update_objects = [o for o in objects if o != None and o.ident != None] - for obj in update_objects: - self.api.update_file(obj.ident, obj, editgroup_id=editgroup_id) + if len(update_objects): + update_eg = self._editgroup().editgroup_id + for obj in update_objects: + self.api.update_file(obj.ident, obj, editgroup_id=update_eg) + self.api.accept_editgroup(update_eg) if len(new_objects) > 0: - self.api.create_file_batch(new_objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_file_batch(new_objects, autoaccept="true") self.counts['update'] += len(update_objects) self.counts['insert'] += len(new_objects) diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index 0aa4ab00..0c8b1d62 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -74,10 +74,10 @@ class OrcidImporter(FatcatImporter): self.api.create_creator(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_orcid_dict(json.loads(l)) for l in batch if l != None] objects = [o for o in objects if o != None] - self.api.create_creator_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_creator_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) diff --git a/python/fatcat_web/templates/editor_changelog.html b/python/fatcat_web/templates/editor_editgroups.html index 785c19bd..785c19bd 100644 --- a/python/fatcat_web/templates/editor_changelog.html +++ b/python/fatcat_web/templates/editor_editgroups.html |