summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--README.md6
-rw-r--r--TODO29
-rw-r--r--python/README_import.md4
-rw-r--r--python/tests/api_annotations.py39
-rw-r--r--python/tests/api_editgroups.py116
-rw-r--r--python/tests/fixtures.py30
-rw-r--r--rust/src/bin/fatcatd.rs1
-rw-r--r--rust/src/endpoints.rs10
8 files changed, 184 insertions, 51 deletions
diff --git a/README.md b/README.md
index 6ed45cb1..8fbaf953 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ schema, lives under `./python_client/`.
- [x] JSON(B) "extra" metadata fields
- [x] full rev1 schema for all entities
- [x] file sets and web captures
- - [ ] editgroup review: comments? actions?
+ - [x] editgroup review: annotations
- HTTP API Server
- [x] base32 encoding of UUID identifiers
- [x] inverse many-to-many helpers (files-by-release, release-by-creator)
@@ -62,6 +62,6 @@ schema, lives under `./python_client/`.
- [x] Basic logging
- [x] Swagger-UI
- [x] Bulk metadata exports
- - [ ] Sentry (error reporting)
- - [ ] Metrics
+ - [x] Sentry (error reporting)
+ - [x] Metrics
diff --git a/TODO b/TODO
index 6417668d..5075f10a 100644
--- a/TODO
+++ b/TODO
@@ -1,32 +1,24 @@
## In Progress
-- basic python tests for editgroup, annotation, submission changes
-- python tests for new autoaccept behavior
-- python tests for citation table storage efficiency changes
- => should there be a distinction between empty list and no references?
- yes, eg if expanded or not hidden
- => postgres manual checks that this is working
- => also benchmark (both speed and efficiency)
+- check that any needed/new indices are in place
+ => seems to at least superficially work
+- benchmark citation efficiency (in QA)
+
+- all query params need to be strings, and parse in rust :(
+ since=(datetime.datetime.utcnow() + datetime.timedelta(seconds=1)).isoformat()+"Z"
+- doc: python client API needs to have booleans set as, eg, 'true'/'false' (str) (!?!?)
+ "note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response"
## Next Up
- "don't clobber" mode/flag for crossref import (and others?)
-- update_file requires 'id'. should it be 'ident'?
- => something different about file vs. release
-- guide updates for auth
-- refactor webface views to use shared entity_view.html template
-- handle 'wip' status entities in web UI
- elastic inserter should handle deletions and redirects; if state isn't
active, delete the document
=> don't delete, just store state. but need to "blank" redirects and WIP so
they don't show up in results
=> refactor inserter to be a class (eg, for command line use)
=> end-to-end test of this behavior?
-- date handling is really pretty bad for releases; mangling those Jan1/Dec31
- => elastic schema should have a year field (integer)
-- document: elastic query date syntax is like: date:[2018-10-01 TO 2018-12-31]
-- elastic transform should only include authors, not editors (?)
- webcapture timestamp schema cleanup (both CDX and base)
=> dt.to_rfc3339_opts(SecondsFormat::Secs, true)
=> but this is mostly buried in serialization code?
@@ -43,6 +35,9 @@
- handle very large author/reference lists (instead of dropping)
=> https://api.crossref.org/v1/works/http://dx.doi.org/10.1007/978-3-319-46095-6_7
=> 7000+ authors (!)
+- guide updates for auth
+- refactor webface views to use shared entity_view.html template
+- handle 'wip' status entities in web UI
## Bugs (or at least need tests)
@@ -151,6 +146,7 @@ new importers:
## Schema / Entity Fields
+- elastic transform should only include authors, not editors (?)
- arxiv_id field (keep flip-flopping)
- original_title field (internationalization, "original language")
- `doi` field for containers (at least for "journal" type; maybe for "series"
@@ -162,6 +158,7 @@ new importers:
## Other / Backburner
+- document: elastic query date syntax is like: date:[2018-10-01 TO 2018-12-31]
- fileset/webcapture webface anything
- display abstracts better. no hashes or metadata; prefer plain or HTML,
convert JATS if necessary
diff --git a/python/README_import.md b/python/README_import.md
index cc9a94e1..9dda725d 100644
--- a/python/README_import.md
+++ b/python/README_import.md
@@ -26,11 +26,13 @@ the others:
wget https://archive.org/download/ia_papers_manifest_2018-01-25/index/idents_files_urls.sqlite.gz
wget https://archive.org/download/ia_journal_metadata_explore_2018-04-05/journal_extra_metadata.csv
wget https://archive.org/download/issn_issnl_mappings/20180216.ISSN-to-ISSN-L.txt
- wget https://archive.org/download/orcid-dump-2017/public_profiles_API-2.0_2017_10_json.tar.gz
+ wget https://archive.org/download/orcid-dump-2017/public_profiles_1_2_json.all.json.gz
wget https://archive.org/download/ia_journal_pid_map_munge_20180908/release_ids.ia_munge_20180908.sqlite3.gz
wget https://archive.org/download/ia_test_paper_matches/2018-08-27-2352.17-matchcrossref.insertable.json.gz
wget https://archive.org/download/ia_papers_manifest_2018-01-25_matched/ia_papers_manifest_2018-01-25.matched.json.gz
+ gunzip public_profiles_1_2_json.all.json.gz
+
## ISSN
From CSV file:
diff --git a/python/tests/api_annotations.py b/python/tests/api_annotations.py
new file mode 100644
index 00000000..0d3c5046
--- /dev/null
+++ b/python/tests/api_annotations.py
@@ -0,0 +1,39 @@
+
+import json
+import pytest
+from copy import copy
+
+from fatcat_client import *
+from fatcat_client.rest import ApiException
+from fixtures import *
+
+
+def test_annotations(api):
+
+ eg = quick_eg(api)
+
+ # ensure no annotations on this object
+ a = api.get_editgroup_annotations(eg.editgroup_id)
+ assert a == []
+
+ # create an annotation!
+ api.create_editgroup_annotation(
+ eg.editgroup_id,
+ EditgroupAnnotation(
+ comment_markdown="some *annotation*",
+ extra=dict(thing="thang")))
+
+ # check that we can fetch it all sorts of ways
+ a = api.get_editgroup_annotations(eg.editgroup_id)
+ assert len(a) == 1
+ assert a[0].extra['thing'] == "thang"
+
+ # the editor persists, so this is a hack to find a "recent" one
+ a2 = api.get_editor_annotations(eg.editor_id, limit=100)
+ found = None
+ for thing in a2:
+ if thing.annotation_id == a[0].annotation_id:
+ found = thing
+ break
+ assert thing
+ assert thing.extra['thing'] == "thang"
diff --git a/python/tests/api_editgroups.py b/python/tests/api_editgroups.py
new file mode 100644
index 00000000..af55cfeb
--- /dev/null
+++ b/python/tests/api_editgroups.py
@@ -0,0 +1,116 @@
+
+import json
+import pytest
+import datetime
+from copy import copy
+
+from fatcat_client import *
+from fatcat_client.rest import ApiException
+from fixtures import *
+
+
+def test_editgroup_submit(api):
+ # 1. check that edit group can be submitted/unsubmitted, and shows up in reviewable appropriately
+ # 2. accepted edits don't show up as reviewable and can't be submitted
+
+ c1 = CreatorEntity(display_name="test updates")
+ eg = quick_eg(api)
+ c1 = api.get_creator(api.create_creator(c1, editgroup_id=eg.editgroup_id).ident)
+
+ eg2 = api.get_editgroup(eg.editgroup_id)
+ assert not eg2.submitted
+ assert not eg2.changelog_index
+
+ reviewable = api.get_editgroups_reviewable(limit=100)
+ assert eg.editgroup_id not in [v.editgroup_id for v in reviewable]
+ wip = api.get_editor_editgroups(eg.editor_id, limit=100)
+ assert eg.editgroup_id in [v.editgroup_id for v in wip]
+
+ api.update_editgroup(eg.editgroup_id, eg2, submit='true')
+ eg3 = api.get_editgroup(eg.editgroup_id)
+ assert eg3.submitted
+ reviewable = api.get_editgroups_reviewable(limit=100)
+ assert eg.editgroup_id in [v.editgroup_id for v in reviewable]
+
+ api.update_editgroup(eg.editgroup_id, eg2, submit='false')
+ eg3 = api.get_editgroup(eg.editgroup_id)
+ assert not eg3.submitted
+ reviewable = api.get_editgroups_reviewable(limit=100)
+ assert eg.editgroup_id not in [v.editgroup_id for v in reviewable]
+
+ # put back in reviewable
+ api.update_editgroup(eg.editgroup_id, eg2, submit='true')
+ reviewable = api.get_editgroups_reviewable(limit=100)
+ assert eg.editgroup_id in [v.editgroup_id for v in reviewable]
+
+ # shouldn't be reviewable if accepted
+ api.accept_editgroup(eg.editgroup_id)
+ reviewable = api.get_editgroups_reviewable(limit=100)
+ assert eg.editgroup_id not in [v.editgroup_id for v in reviewable]
+ eg3 = api.get_editgroup(eg.editgroup_id)
+ #print(eg3)
+ assert eg3.submitted
+ assert eg3.changelog_index
+
+ with pytest.raises(fatcat_client.rest.ApiException):
+ api.update_editgroup(eg.editgroup_id, eg3, submit='true')
+ with pytest.raises(fatcat_client.rest.ApiException):
+ eg3.description = "something"
+ api.update_editgroup(eg.editgroup_id, eg3)
+
+
+def test_editgroup_ordering(api):
+
+ eg1 = quick_eg(api)
+ eg2 = quick_eg(api)
+ api.update_editgroup(
+ eg1.editgroup_id,
+ Editgroup(editgroup_id=eg1.editgroup_id, description="FAIL"),
+ submit='true')
+ api.update_editgroup(
+ eg2.editgroup_id,
+ Editgroup(editgroup_id=eg2.editgroup_id, description="FAIL"),
+ submit='true')
+
+ r1 = api.get_editgroups_reviewable()
+ #print(r1)
+ assert not r1[0].description
+ assert not r1[1].description
+ assert r1[0].submitted >= r1[1].submitted
+
+ # should be no editgroups "in the future" (since now + 1sec)
+ r1 = api.get_editgroups_reviewable(since=(datetime.datetime.utcnow() + datetime.timedelta(seconds=1)).isoformat()+"Z")
+ assert not r1
+
+ r1 = api.get_editgroups_reviewable(since=(datetime.datetime.utcnow() - datetime.timedelta(seconds=5)).isoformat()+"Z")
+ assert r1[0].submitted <= r1[1].submitted
+
+
+def test_editgroup_autoaccept(api):
+ # autoaccept changes: editgroups required when, in what combination
+
+ eg = quick_eg(api)
+ c1 = CreatorEntity(display_name="test autoaccept")
+ c2 = CreatorEntity(display_name="test another autoaccept")
+
+ with pytest.raises(fatcat_client.rest.ApiException):
+ edits = api.create_creator_batch([c1, c2])
+
+ with pytest.raises(fatcat_client.rest.ApiException):
+ edits = api.create_creator_batch([c1, c2], editgroup_id=eg.editgroup_id, autoaccept='true')
+
+ edits1 = api.create_creator_batch([c1, c2], editgroup_id=eg.editgroup_id)
+ edits2 = api.create_creator_batch([c1, c2], autoaccept='true')
+
+ assert edits1[0].editgroup_id == eg.editgroup_id
+ assert edits1[0].editgroup_id != edits2[1].editgroup_id
+ eg1 = api.get_editgroup(edits1[0].editgroup_id)
+ eg2 = api.get_editgroup(edits2[0].editgroup_id)
+
+ assert not eg1.changelog_index
+ assert eg2.changelog_index
+ #print(edits1)
+ #print(eg1.edits.creators)
+ assert eg1.edits.creators[0].ident == edits1[0].ident
+ assert eg2.edits.creators[0].ident == edits2[0].ident
+
diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py
index 6a880c48..3cc275b3 100644
--- a/python/tests/fixtures.py
+++ b/python/tests/fixtures.py
@@ -28,6 +28,7 @@ def api():
conf.api_key["Authorization"] = os.getenv("FATCAT_API_AUTH_TOKEN")
conf.api_key_prefix["Authorization"] = "Bearer"
api_client = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf))
+ api_client.editor_id = "aaaaaaaaaaaabkvkaaaaaaaaae"
return api_client
def test_get_changelog_entry(api):
@@ -38,33 +39,6 @@ def test_get_changelog_entry(api):
## Helpers ##################################################################
def quick_eg(api_inst):
- eg = api_inst.create_editgroup(
- fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae'))
+ eg = api_inst.create_editgroup(fatcat_client.Editgroup())
return eg
-# TODO: what are these even here for?
-def check_entity_fields(e):
- for key in ('rev', 'is_live', 'redirect_id'):
- assert key in e
- for key in ('id',):
- assert e[key] is not None
-
-def check_release(e):
- for key in ('work', 'release_type'):
- assert key in e
- for key in ('title', ):
- assert e[key] is not None
- for key in ('refs', 'creators'):
- assert type(e[key]) == list
-
-def check_creator(e):
- for key in ('name',):
- assert e[key] is not None
-
-def check_container(e):
- for key in ('name',):
- assert e[key] is not None
-
-def check_file(e):
- for key in ('size', 'sha1'):
- assert e[key] is not None
diff --git a/rust/src/bin/fatcatd.rs b/rust/src/bin/fatcatd.rs
index 75a6f000..ccce6725 100644
--- a/rust/src/bin/fatcatd.rs
+++ b/rust/src/bin/fatcatd.rs
@@ -88,7 +88,6 @@ fn main() -> Result<()> {
server.metrics.incr("restart").unwrap();
}
};
- info!(logger, "{:#?}", server.metrics);
info!(
logger,
diff --git a/rust/src/endpoints.rs b/rust/src/endpoints.rs
index f7e93448..670c7fd9 100644
--- a/rust/src/endpoints.rs
+++ b/rust/src/endpoints.rs
@@ -143,13 +143,19 @@ macro_rules! wrap_entity_handlers {
let conn = self.db_pool.get().expect("db_pool error");
let ret = match conn.transaction(|| {
let auth_context = self.auth_confectionary.require_auth(&conn, &context.auth_data, Some(stringify!($post_batch_fn)))?;
- auth_context.require_role(FatcatRole::Editor)?;
+ let autoaccept = autoaccept.unwrap_or(false);
+ if autoaccept {
+ auth_context.require_role(FatcatRole::Admin)?;
+ } else {
+ auth_context.require_role(FatcatRole::Editor)?;
+ };
let editgroup_id = if let Some(s) = editgroup_id {
+ // make_edit_context() checks for "both editgroup_id and autosubmit" error case
let eg_id = FatcatId::from_str(&s)?;
auth_context.require_editgroup(&conn, eg_id)?;
Some(eg_id)
} else { None };
- self.$post_batch_handler(&conn, entity_list, autoaccept.unwrap_or(false), auth_context.editor_id, editgroup_id)
+ self.$post_batch_handler(&conn, entity_list, autoaccept, auth_context.editor_id, editgroup_id)
}).map_err(|e| FatcatError::from(e)) {
Ok(edits) => {
self.metrics.count("entities.created", edits.len() as i64).ok();