diff options
author | Bryan Newbold <bnewbold@robocracy.org> | 2019-01-11 15:17:32 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-01-11 15:17:34 -0800 |
commit | 811f0a52779afdaa8e1f11a3781fc22fa3acc02c (patch) | |
tree | e6bed28dfde8e897d97b85fabfc731dc0a9e18fb /python/fatcat_tools/importers | |
parent | ee938e9edc9abfc33cdef6dc716eff89aceb5ca6 (diff) | |
download | fatcat-811f0a52779afdaa8e1f11a3781fc22fa3acc02c.tar.gz fatcat-811f0a52779afdaa8e1f11a3781fc22fa3acc02c.zip |
use full-on autoaccept mode
Now that editor_id is infered from token, don't *need* to create ahead
of time. This backend change simplifies things greatly (either update an
existing editgroup, or create new and *only* include entities in the
batch transaction), at the cost of being able to configure the editgroup
in any way, including setting a description.
Diffstat (limited to 'python/fatcat_tools/importers')
-rw-r--r-- | python/fatcat_tools/importers/common.py | 5 | ||||
-rw-r--r-- | python/fatcat_tools/importers/crossref.py | 4 | ||||
-rw-r--r-- | python/fatcat_tools/importers/issn.py | 4 | ||||
-rw-r--r-- | python/fatcat_tools/importers/matched.py | 11 | ||||
-rw-r--r-- | python/fatcat_tools/importers/orcid.py | 4 |
5 files changed, 16 insertions, 12 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index e39ec6c9..06897bee 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -96,8 +96,9 @@ class FatcatImporter: if decode_kafka: rows = [msg.value.decode('utf-8') for msg in rows] self.counts['processed_lines'] += len(rows) - eg = self._editgroup() - self.create_batch(rows, editgroup_id=eg.editgroup_id) + #eg = self._editgroup() + #self.create_batch(rows, editgroup_id=eg.editgroup_id) + self.create_batch(rows) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py index ed60a78c..6365e491 100644 --- a/python/fatcat_tools/importers/crossref.py +++ b/python/fatcat_tools/importers/crossref.py @@ -313,7 +313,7 @@ class CrossrefImporter(FatcatImporter): self.api.create_release(re, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Current work/release pairing disallows batch creation of releases. Could do batch work creation and then match against releases, but meh.""" release_batch = [] @@ -331,5 +331,5 @@ class CrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_release_batch(release_batch, autoaccept="true") self.counts['insert'] += len(release_batch) diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py index 02a1eea0..f4d525a4 100644 --- a/python/fatcat_tools/importers/issn.py +++ b/python/fatcat_tools/importers/issn.py @@ -80,10 +80,10 @@ class IssnImporter(FatcatImporter): self.api.create_container(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_issn_row(l) for l in batch if (l is not None)] objects = [o for o in objects if (o is not None)] - self.api.create_container_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_container_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py index 0b77bcf0..1e5c22f7 100644 --- a/python/fatcat_tools/importers/matched.py +++ b/python/fatcat_tools/importers/matched.py @@ -142,15 +142,18 @@ class MatchedImporter(FatcatImporter): self.api.update_file(fe.ident, fe, editgroup_id=editgroup_id) self.counts['update'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_matched_dict(json.loads(l)) for l in batch if l != None] new_objects = [o for o in objects if o != None and o.ident == None] update_objects = [o for o in objects if o != None and o.ident != None] - for obj in update_objects: - self.api.update_file(obj.ident, obj, editgroup_id=editgroup_id) + if len(update_objects): + update_eg = self._editgroup().editgroup_id + for obj in update_objects: + self.api.update_file(obj.ident, obj, editgroup_id=update_eg) + self.api.accept_editgroup(update_eg) if len(new_objects) > 0: - self.api.create_file_batch(new_objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_file_batch(new_objects, autoaccept="true") self.counts['update'] += len(update_objects) self.counts['insert'] += len(new_objects) diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index 0aa4ab00..0c8b1d62 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -74,10 +74,10 @@ class OrcidImporter(FatcatImporter): self.api.create_creator(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_orcid_dict(json.loads(l)) for l in batch if l != None] objects = [o for o in objects if o != None] - self.api.create_creator_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_creator_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) |