summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-01-11 15:17:32 -0800
committerBryan Newbold <bnewbold@robocracy.org>2019-01-11 15:17:34 -0800
commit811f0a52779afdaa8e1f11a3781fc22fa3acc02c (patch)
treee6bed28dfde8e897d97b85fabfc731dc0a9e18fb /python/fatcat_tools
parentee938e9edc9abfc33cdef6dc716eff89aceb5ca6 (diff)
downloadfatcat-811f0a52779afdaa8e1f11a3781fc22fa3acc02c.tar.gz
fatcat-811f0a52779afdaa8e1f11a3781fc22fa3acc02c.zip
use full-on autoaccept mode
Now that editor_id is infered from token, don't *need* to create ahead of time. This backend change simplifies things greatly (either update an existing editgroup, or create new and *only* include entities in the batch transaction), at the cost of being able to configure the editgroup in any way, including setting a description.
Diffstat (limited to 'python/fatcat_tools')
-rw-r--r--python/fatcat_tools/importers/common.py5
-rw-r--r--python/fatcat_tools/importers/crossref.py4
-rw-r--r--python/fatcat_tools/importers/issn.py4
-rw-r--r--python/fatcat_tools/importers/matched.py11
-rw-r--r--python/fatcat_tools/importers/orcid.py4
5 files changed, 16 insertions, 12 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index e39ec6c9..06897bee 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -96,8 +96,9 @@ class FatcatImporter:
if decode_kafka:
rows = [msg.value.decode('utf-8') for msg in rows]
self.counts['processed_lines'] += len(rows)
- eg = self._editgroup()
- self.create_batch(rows, editgroup_id=eg.editgroup_id)
+ #eg = self._editgroup()
+ #self.create_batch(rows, editgroup_id=eg.editgroup_id)
+ self.create_batch(rows)
def process_csv_source(self, source, group_size=100, delimiter=','):
reader = csv.DictReader(source, delimiter=delimiter)
diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py
index ed60a78c..6365e491 100644
--- a/python/fatcat_tools/importers/crossref.py
+++ b/python/fatcat_tools/importers/crossref.py
@@ -313,7 +313,7 @@ class CrossrefImporter(FatcatImporter):
self.api.create_release(re, editgroup_id=editgroup_id)
self.counts['insert'] += 1
- def create_batch(self, batch, editgroup_id=None):
+ def create_batch(self, batch):
"""Current work/release pairing disallows batch creation of releases.
Could do batch work creation and then match against releases, but meh."""
release_batch = []
@@ -331,5 +331,5 @@ class CrossrefImporter(FatcatImporter):
re.container_id = container.ident
self._issnl_id_map[ce.issnl] = container.ident
release_batch.append(re)
- self.api.create_release_batch(release_batch, autoaccept="true", editgroup_id=editgroup_id)
+ self.api.create_release_batch(release_batch, autoaccept="true")
self.counts['insert'] += len(release_batch)
diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py
index 02a1eea0..f4d525a4 100644
--- a/python/fatcat_tools/importers/issn.py
+++ b/python/fatcat_tools/importers/issn.py
@@ -80,10 +80,10 @@ class IssnImporter(FatcatImporter):
self.api.create_container(ce, editgroup_id=editgroup_id)
self.counts['insert'] += 1
- def create_batch(self, batch, editgroup_id=None):
+ def create_batch(self, batch):
"""Reads and processes in batches (not API-call-per-line)"""
objects = [self.parse_issn_row(l)
for l in batch if (l is not None)]
objects = [o for o in objects if (o is not None)]
- self.api.create_container_batch(objects, autoaccept="true", editgroup_id=editgroup_id)
+ self.api.create_container_batch(objects, autoaccept="true")
self.counts['insert'] += len(objects)
diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py
index 0b77bcf0..1e5c22f7 100644
--- a/python/fatcat_tools/importers/matched.py
+++ b/python/fatcat_tools/importers/matched.py
@@ -142,15 +142,18 @@ class MatchedImporter(FatcatImporter):
self.api.update_file(fe.ident, fe, editgroup_id=editgroup_id)
self.counts['update'] += 1
- def create_batch(self, batch, editgroup_id=None):
+ def create_batch(self, batch):
"""Reads and processes in batches (not API-call-per-line)"""
objects = [self.parse_matched_dict(json.loads(l))
for l in batch if l != None]
new_objects = [o for o in objects if o != None and o.ident == None]
update_objects = [o for o in objects if o != None and o.ident != None]
- for obj in update_objects:
- self.api.update_file(obj.ident, obj, editgroup_id=editgroup_id)
+ if len(update_objects):
+ update_eg = self._editgroup().editgroup_id
+ for obj in update_objects:
+ self.api.update_file(obj.ident, obj, editgroup_id=update_eg)
+ self.api.accept_editgroup(update_eg)
if len(new_objects) > 0:
- self.api.create_file_batch(new_objects, autoaccept="true", editgroup_id=editgroup_id)
+ self.api.create_file_batch(new_objects, autoaccept="true")
self.counts['update'] += len(update_objects)
self.counts['insert'] += len(new_objects)
diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py
index 0aa4ab00..0c8b1d62 100644
--- a/python/fatcat_tools/importers/orcid.py
+++ b/python/fatcat_tools/importers/orcid.py
@@ -74,10 +74,10 @@ class OrcidImporter(FatcatImporter):
self.api.create_creator(ce, editgroup_id=editgroup_id)
self.counts['insert'] += 1
- def create_batch(self, batch, editgroup_id=None):
+ def create_batch(self, batch):
"""Reads and processes in batches (not API-call-per-line)"""
objects = [self.parse_orcid_dict(json.loads(l))
for l in batch if l != None]
objects = [o for o in objects if o != None]
- self.api.create_creator_batch(objects, autoaccept="true", editgroup_id=editgroup_id)
+ self.api.create_creator_batch(objects, autoaccept="true")
self.counts['insert'] += len(objects)