From 811f0a52779afdaa8e1f11a3781fc22fa3acc02c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 11 Jan 2019 15:17:32 -0800 Subject: use full-on autoaccept mode Now that editor_id is infered from token, don't *need* to create ahead of time. This backend change simplifies things greatly (either update an existing editgroup, or create new and *only* include entities in the batch transaction), at the cost of being able to configure the editgroup in any way, including setting a description. --- python/fatcat_tools/importers/common.py | 5 ++-- python/fatcat_tools/importers/crossref.py | 4 +-- python/fatcat_tools/importers/issn.py | 4 +-- python/fatcat_tools/importers/matched.py | 11 +++++--- python/fatcat_tools/importers/orcid.py | 4 +-- python/fatcat_web/templates/editor_changelog.html | 29 ---------------------- python/fatcat_web/templates/editor_editgroups.html | 29 ++++++++++++++++++++++ 7 files changed, 45 insertions(+), 41 deletions(-) delete mode 100644 python/fatcat_web/templates/editor_changelog.html create mode 100644 python/fatcat_web/templates/editor_editgroups.html (limited to 'python') diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index e39ec6c9..06897bee 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -96,8 +96,9 @@ class FatcatImporter: if decode_kafka: rows = [msg.value.decode('utf-8') for msg in rows] self.counts['processed_lines'] += len(rows) - eg = self._editgroup() - self.create_batch(rows, editgroup_id=eg.editgroup_id) + #eg = self._editgroup() + #self.create_batch(rows, editgroup_id=eg.editgroup_id) + self.create_batch(rows) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py index ed60a78c..6365e491 100644 --- a/python/fatcat_tools/importers/crossref.py +++ b/python/fatcat_tools/importers/crossref.py @@ -313,7 +313,7 @@ class CrossrefImporter(FatcatImporter): self.api.create_release(re, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Current work/release pairing disallows batch creation of releases. Could do batch work creation and then match against releases, but meh.""" release_batch = [] @@ -331,5 +331,5 @@ class CrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_release_batch(release_batch, autoaccept="true") self.counts['insert'] += len(release_batch) diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py index 02a1eea0..f4d525a4 100644 --- a/python/fatcat_tools/importers/issn.py +++ b/python/fatcat_tools/importers/issn.py @@ -80,10 +80,10 @@ class IssnImporter(FatcatImporter): self.api.create_container(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_issn_row(l) for l in batch if (l is not None)] objects = [o for o in objects if (o is not None)] - self.api.create_container_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_container_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py index 0b77bcf0..1e5c22f7 100644 --- a/python/fatcat_tools/importers/matched.py +++ b/python/fatcat_tools/importers/matched.py @@ -142,15 +142,18 @@ class MatchedImporter(FatcatImporter): self.api.update_file(fe.ident, fe, editgroup_id=editgroup_id) self.counts['update'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_matched_dict(json.loads(l)) for l in batch if l != None] new_objects = [o for o in objects if o != None and o.ident == None] update_objects = [o for o in objects if o != None and o.ident != None] - for obj in update_objects: - self.api.update_file(obj.ident, obj, editgroup_id=editgroup_id) + if len(update_objects): + update_eg = self._editgroup().editgroup_id + for obj in update_objects: + self.api.update_file(obj.ident, obj, editgroup_id=update_eg) + self.api.accept_editgroup(update_eg) if len(new_objects) > 0: - self.api.create_file_batch(new_objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_file_batch(new_objects, autoaccept="true") self.counts['update'] += len(update_objects) self.counts['insert'] += len(new_objects) diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index 0aa4ab00..0c8b1d62 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -74,10 +74,10 @@ class OrcidImporter(FatcatImporter): self.api.create_creator(ce, editgroup_id=editgroup_id) self.counts['insert'] += 1 - def create_batch(self, batch, editgroup_id=None): + def create_batch(self, batch): """Reads and processes in batches (not API-call-per-line)""" objects = [self.parse_orcid_dict(json.loads(l)) for l in batch if l != None] objects = [o for o in objects if o != None] - self.api.create_creator_batch(objects, autoaccept="true", editgroup_id=editgroup_id) + self.api.create_creator_batch(objects, autoaccept="true") self.counts['insert'] += len(objects) diff --git a/python/fatcat_web/templates/editor_changelog.html b/python/fatcat_web/templates/editor_changelog.html deleted file mode 100644 index 785c19bd..00000000 --- a/python/fatcat_web/templates/editor_changelog.html +++ /dev/null @@ -1,29 +0,0 @@ -{% extends "base.html" %} -{% block body %} - -

Editor Changelog: {{ editor.username }} -
- - editor {{ editor.editor_id }} - -
-

- -

Changes accepted (aka, merged editgroups): - - - {% for entry in changelog_entries %} -
Changelog
Index - 		Timestamp (UTC) - Editgroup - Editor - Description -
{{ entry.index }} - {{ entry.timestamp }} - {{ entry.editgroup_id }} - {{ entry.editgroup.editor_id }} - {% if entry.editgroup.description != None %}{{ entry.editgroup.description }}{% endif %} - {% endfor %} -
- -{% endblock %} diff --git a/python/fatcat_web/templates/editor_editgroups.html b/python/fatcat_web/templates/editor_editgroups.html new file mode 100644 index 00000000..785c19bd --- /dev/null +++ b/python/fatcat_web/templates/editor_editgroups.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} +{% block body %} + +

Editor Changelog: {{ editor.username }} +
+ + editor {{ editor.editor_id }} + +
+

+ +

Changes accepted (aka, merged editgroups): + + + {% for entry in changelog_entries %} +
Changelog
Index + 		Timestamp (UTC) + Editgroup + Editor + Description +
{{ entry.index }} + {{ entry.timestamp }} + {{ entry.editgroup_id }} + {{ entry.editgroup.editor_id }} + {% if entry.editgroup.description != None %}{{ entry.editgroup.description }}{% endif %} + {% endfor %} +
+ +{% endblock %} -- cgit v1.2.3