diff options
22 files changed, 953 insertions, 372 deletions
diff --git a/extra/sql_dumps/README.md b/extra/sql_dumps/README.md index ad4afa8f..bb41e702 100644 --- a/extra/sql_dumps/README.md +++ b/extra/sql_dumps/README.md @@ -83,7 +83,7 @@ Or, in production: sudo su postgres time zcat fatcat_full_dbdump_2020-02-02.022209.tar.gz | pg_restore --exit-on-error --clean --if-exists --dbname fatcat_prod -In QA (as of 2019-01-30 dump), this takes about 5h15m. +In QA (as of 2019-12-13 dump), this takes about 8 hours. To just inspect a dump: diff --git a/extra/stats/2019-12-20-prod-stats.json b/extra/stats/2019-12-20-prod-stats.json new file mode 100644 index 00000000..bf2b3eb2 --- /dev/null +++ b/extra/stats/2019-12-20-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":3046251,"timestamp":"2019-12-21T01:07:12.968312+00:00"}},"container":{"total":144176},"papers":{"in_kbart":60298077,"in_web":18237653,"in_web_not_kbart":8312648,"is_oa":10401008,"total":100217831},"release":{"refs_total":801902579,"total":124161369}} diff --git a/extra/stats/2019-12-20-prod-table-sizes.txt b/extra/stats/2019-12-20-prod-table-sizes.txt new file mode 100644 index 00000000..a778e6f7 --- /dev/null +++ b/extra/stats/2019-12-20-prod-table-sizes.txt @@ -0,0 +1,147 @@ + table_name | table_size | indexes_size | total_size +--------------------------------------------------------------+------------+--------------+------------ + "public"."refs_blob" | 83 GB | 2873 MB | 85 GB + "public"."release_rev" | 47 GB | 29 GB | 76 GB + "public"."release_contrib" | 35 GB | 31 GB | 66 GB + "public"."release_edit" | 12 GB | 19 GB | 31 GB + "public"."work_edit" | 12 GB | 19 GB | 30 GB + "public"."release_ident" | 8218 MB | 14 GB | 22 GB + "public"."work_ident" | 8021 MB | 14 GB | 22 GB + "public"."abstracts" | 12 GB | 916 MB | 13 GB + "public"."file_rev_url" | 9364 MB | 3465 MB | 13 GB + "public"."work_rev" | 5205 MB | 5825 MB | 11 GB + "public"."release_ref" | 3950 MB | 5626 MB | 9577 MB + "public"."file_rev" | 3062 MB | 4431 MB | 7494 MB + "public"."file_edit" | 2524 MB | 3798 MB | 6322 MB + "public"."file_ident" | 1662 MB | 2357 MB | 4019 MB + "public"."file_rev_release" | 1533 MB | 2387 MB | 3920 MB + "public"."release_rev_abstract" | 1339 MB | 1678 MB | 3017 MB + "public"."editgroup" | 653 MB | 365 MB | 1018 MB + "public"."creator_edit" | 377 MB | 420 MB | 797 MB + "public"."creator_rev" | 370 MB | 353 MB | 723 MB + "public"."creator_ident" | 254 MB | 319 MB | 573 MB + "public"."release_rev_extid" | 190 MB | 296 MB | 486 MB + "public"."changelog" | 184 MB | 204 MB | 388 MB + "public"."container_rev" | 74 MB | 22 MB | 96 MB + "public"."container_edit" | 24 MB | 30 MB | 54 MB + "public"."container_ident" | 10 MB | 19 MB | 29 MB + "pg_catalog"."pg_statistic" | 1448 kB | 40 kB | 1488 kB + "pg_catalog"."pg_depend" | 568 kB | 848 kB | 1416 kB + "pg_catalog"."pg_proc" | 648 kB | 352 kB | 1000 kB + "pg_catalog"."pg_attribute" | 664 kB | 288 kB | 952 kB + "pg_catalog"."pg_rewrite" | 584 kB | 32 kB | 616 kB + "pg_catalog"."pg_description" | 360 kB | 184 kB | 544 kB + "pg_catalog"."pg_class" | 160 kB | 136 kB | 296 kB + "pg_catalog"."pg_collation" | 216 kB | 80 kB | 296 kB + "pg_catalog"."pg_constraint" | 152 kB | 128 kB | 280 kB + "pg_catalog"."pg_operator" | 152 kB | 88 kB | 240 kB + "pg_catalog"."pg_type" | 144 kB | 80 kB | 224 kB + "pg_catalog"."pg_amop" | 88 kB | 112 kB | 200 kB + "pg_catalog"."pg_trigger" | 80 kB | 64 kB | 144 kB + "pg_catalog"."pg_amproc" | 64 kB | 64 kB | 128 kB + "pg_catalog"."pg_index" | 80 kB | 32 kB | 112 kB + "pg_catalog"."pg_conversion" | 56 kB | 48 kB | 104 kB + "public"."webcapture_rev_cdx" | 64 kB | 32 kB | 96 kB + "information_schema"."sql_features" | 96 kB | 0 bytes | 96 kB + "pg_catalog"."pg_attrdef" | 56 kB | 32 kB | 88 kB + "pg_catalog"."pg_opclass" | 56 kB | 32 kB | 88 kB + "pg_catalog"."pg_cast" | 48 kB | 32 kB | 80 kB + "pg_catalog"."pg_opfamily" | 48 kB | 32 kB | 80 kB + "pg_catalog"."pg_ts_config_map" | 48 kB | 32 kB | 80 kB + "public"."fileset_rev_file" | 48 kB | 32 kB | 80 kB + "pg_catalog"."pg_am" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_language" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_namespace" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_database" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_tablespace" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_auth_members" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_shdepend" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_ts_config" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_ts_dict" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_ts_parser" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_ts_template" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_extension" | 40 kB | 32 kB | 72 kB + "pg_catalog"."pg_authid" | 40 kB | 32 kB | 72 kB + "public"."editor" | 16 kB | 48 kB | 64 kB + "public"."editgroup_annotation" | 16 kB | 48 kB | 64 kB + "public"."auth_oidc" | 16 kB | 48 kB | 64 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "pg_catalog"."pg_init_privs" | 48 kB | 16 kB | 64 kB + "public"."webcapture_edit" | 16 kB | 48 kB | 64 kB + "pg_catalog"."pg_shdescription" | 48 kB | 16 kB | 64 kB + "pg_catalog"."pg_aggregate" | 48 kB | 16 kB | 64 kB + "pg_catalog"."pg_range" | 40 kB | 16 kB | 56 kB + "pg_catalog"."pg_pltemplate" | 40 kB | 16 kB | 56 kB + "information_schema"."sql_sizing" | 48 kB | 0 bytes | 48 kB + "information_schema"."sql_packages" | 48 kB | 0 bytes | 48 kB + "information_schema"."sql_languages" | 48 kB | 0 bytes | 48 kB + "information_schema"."sql_implementation_info" | 48 kB | 0 bytes | 48 kB + "public"."webcapture_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "information_schema"."sql_parts" | 48 kB | 0 bytes | 48 kB + "public"."webcapture_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."webcapture_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."webcapture_rev" | 16 kB | 16 kB | 32 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "pg_catalog"."pg_statistic_ext" | 8192 bytes | 24 kB | 32 kB + "pg_catalog"."pg_sequence" | 8192 bytes | 16 kB | 24 kB + "pg_catalog"."pg_enum" | 0 bytes | 24 kB | 24 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB + "pg_catalog"."pg_inherits" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_user_mapping" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_subscription" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_shseclabel" | 8192 bytes | 8192 bytes | 16 kB + "pg_catalog"."pg_transform" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_seclabel" | 8192 bytes | 8192 bytes | 16 kB + "pg_catalog"."pg_publication" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_publication_rel" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_default_acl" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_replication_origin" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_policy" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_foreign_server" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_event_trigger" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_foreign_data_wrapper" | 0 bytes | 16 kB | 16 kB + "pg_catalog"."pg_db_role_setting" | 8192 bytes | 8192 bytes | 16 kB + "pg_catalog"."pg_largeobject_metadata" | 0 bytes | 8192 bytes | 8192 bytes + "pg_catalog"."pg_largeobject" | 0 bytes | 8192 bytes | 8192 bytes + "pg_catalog"."pg_partitioned_table" | 0 bytes | 8192 bytes | 8192 bytes + "pg_catalog"."pg_subscription_rel" | 0 bytes | 8192 bytes | 8192 bytes + "pg_catalog"."pg_foreign_table" | 0 bytes | 8192 bytes | 8192 bytes + "information_schema"."sql_sizing_profiles" | 8192 bytes | 0 bytes | 8192 bytes + "pg_catalog"."pg_stat_xact_user_tables" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_all_tables" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_sys_tables" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_user_tables" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_all_indexes" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_sys_indexes" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_user_indexes" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_user_indexes" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_all_sequences" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_sys_sequences" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_statio_user_sequences" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_activity" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_replication" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_wal_receiver" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_subscription" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_ssl" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_replication_slots" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_database" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_database_conflicts" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_user_functions" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_xact_user_functions" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_archiver" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_bgwriter" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_stat_progress_vacuum" | 0 bytes | 0 bytes | 0 bytes + "pg_catalog"."pg_user_mappings" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."key_column_usage" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."information_schema_catalog_name" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."column_privileges" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."applicable_roles" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."administrable_role_authorizations" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."domain_udt_usage" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."attributes" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."column_udt_usage" | 0 bytes | 0 bytes | 0 bytes + "information_schema"."character_sets" | 0 bytes | 0 bytes | 0 bytes +[...] diff --git a/guide/src/entity_release.md b/guide/src/entity_release.md index 8d2930d7..242a3c72 100644 --- a/guide/src/entity_release.md +++ b/guide/src/entity_release.md @@ -17,7 +17,7 @@ - `release_type` (string, controlled set): represents the medium or form-factor of this release; eg, "book" versus "journal article". Not necessarily the same across all releases of a work. See definitions below. -- `release_state` (string, controlled set): represents the publishing/review +- `release_stage` (string, controlled set): represents the publishing/review lifecycle status of this particular release of the work. See definitions below. - `release_date` (string, ISO date format): when this release was first made @@ -266,7 +266,7 @@ For the purpose of statistics, the following release types are considered - `paper-conference` - `thesis` -#### `release_state` Vocabulary +#### `release_stage` Vocabulary These roughly follow the [DRIVER](http://web.archive.org/web/20091109125137/http://www2.lse.ac.uk/library/versions/VERSIONS_Toolkit_v1_final.pdf) publication version guidelines, with the addition of a `retracted` status. diff --git a/notes/bulk_edits/2019-12-20_updates.md b/notes/bulk_edits/2019-12-20_updates.md new file mode 100644 index 00000000..a8f62ea9 --- /dev/null +++ b/notes/bulk_edits/2019-12-20_updates.md @@ -0,0 +1,82 @@ + +## Arxiv + +Used metha-sync tool to update. Then went in raw storage directory (as opposed +to using `metha-cat`) and plucked out weekly files updated since last import. +Created a tarball and uploaded to: + + https://archive.org/download/arxiv_raw_oai_snapshot_2019-05-22/arxiv_20190522_20191220.tar.gz + +Downloaded, extracted, then unzipped: + + gunzip *.gz + +Run importer: + + export FATCAT_AUTH_WORKER_ARXIV=... + + ./fatcat_import.py --batch-size 100 arxiv /srv/fatcat/datasets/arxiv_20190522_20191220/2019-05-31-00000000.xml + # Counter({'exists': 1785, 'total': 1001, 'insert': 549, 'skip': 1, 'update': 0}) + + fd .xml /srv/fatcat/datasets/arxiv_20190522_20191220/ | parallel -j15 ./fatcat_import.py --batch-size 100 arxiv {} + +Things seem to run smoothly in QA. New releases get grouped with old works +correctly, no duplication obvious. + +In prod, loaded just the first file as a start, waiting to see if auto-ingest +happens. Looks like yes! Great that everything is so smooth. All seem to be new +captures. + +In production prod elasticsearch, 2,377,645 arxiv releases before this +updated import, 741,033 with files attached. Guessing about 150k new releases, +but will check. + +Up to 2,531,542 arxiv releases, so only 154k or so new releases created. +781,122 with fulltext. + +## Pubmed + +Grabbed fresh 2020 baseline, released in December 2019: <https://archive.org/details/pubmed_medline_baseline_2020> + + gunzip *.xml.gz + +Run importer: + + export FATCAT_AUTH_WORKER_PUBMED=... + + ./fatcat_import.py pubmed /srv/fatcat/datasets/pubmed_medline_baseline_2020/pubmed20n1000.xml /srv/fatcat/datasets/ISSN-to-ISSN-L.txt + + # Counter({'total': 29975, 'update': 26650, 'skip': 2081, 'insert': 1193, 'warn-pmid-doi-mismatch': 36, 'exists': 36, 'skip-update-conflict': 15, 'inserted.container': 3}) + +Noticed that `release_year` was not getting set for many releases. Made a small +code tweak (`1bb0a2181d5a30241d80279c5930eb753733f30b`) and trying another: + + time ./fatcat_import.py pubmed /srv/fatcat/datasets/pubmed_medline_baseline_2020/pubmed20n1001.xml /srv/fatcat/datasets/ISSN-to-ISSN-L.txt + + # Counter({'total': 30000, 'update': 25912, 'skip': 2119, 'insert': 1935, 'exists': 29, 'warn-pmid-doi-mismatch': 27, 'skip-update-conflict': 5, 'inserted.container': 1}) + + real 30m45.044s + user 16m43.672s + sys 0m10.792s + + time fd '.xml$' /srv/fatcat/datasets/pubmed_medline_baseline_2020 | time parallel -j16 ./fatcat_import.py pubmed {} /srv/fatcat/datasets/ISSN-to-ISSN-L.txt + +More errors: + + HTTP response body: {"success":false,"error":"MalformedExternalId","message":"external identifier doesn't match required pattern for a DOI (expected, eg, '10.1234/aksjdfh'): 10.3760/cma. j. issn.2095-4352. 2014. 07.014"} + HTTP response body: {"success":false,"error":"MalformedExternalId","message":"external identifier doesn't match required pattern for a DOI (expected, eg, '10.1234/aksjdfh'): 10.13201/j.issn.10011781.2016.06.002"} + HTTP response body: {"success":false,"error":"MalformedExternalId","message":"external identifier doesn't match required pattern for a DOI (expected, eg, '10.1234/aksjdfh'): 10.23750/abm.v88i2 -s.6506"} + + + 10.1037//0002-9432.72.1.50 + BOGUS DOI: 10.1037//0021-843x.106.2.266 + BOGUS DOI: 10.1037//0021-843x.106.2.280 + => actual ok? at least redirect ok + + unparsable medline date, skipping: Summer 2018 + +TODO: +x fix bad DOI error (real error, skip these) +x remove newline after "unparsable medline date" error +x remove extra line like "existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid))" in warning + diff --git a/notes/bulk_edits/CHANGELOG.md b/notes/bulk_edits/CHANGELOG.md index 3aa89b87..80760938 100644 --- a/notes/bulk_edits/CHANGELOG.md +++ b/notes/bulk_edits/CHANGELOG.md @@ -9,13 +9,24 @@ this file should probably get merged into the guide at some point. This file should not turn in to a TODO list! +## 2019-12 + +Inserted about 154k new arxiv release entities. Still no automatic daily +harvesting. + +"Save Paper Now" importer running. This bot only *submits* editgroups for +review, doesn't auto-accept them. + +## 2019-11 + +Daily ingest of fulltext for OA releases now enabled. New file entities created +and merged automatically. + ## 2019-10 Inserted 1.45m new release entities from Crossref which had been missed during a previous gap in continuous metadata harvesting. -## 2019-10 - Updated 304,308 file entities to remove broken "https://web.archive.org/web/None/*" URLs. diff --git a/python/Pipfile b/python/Pipfile index ae261053..dfb87514 100644 --- a/python/Pipfile +++ b/python/Pipfile @@ -8,10 +8,10 @@ verify_ssl = true name = "pypi" [dev-packages] -pytest = ">=4" +pytest = ">=4,<5.0.0" pytest-pythonpath = "*" pytest-pylint = "*" -ipython = "*" +ipython = "<7.0.0" responses = ">=0.10" pytest-cov = "*" pylint = "*" @@ -29,7 +29,7 @@ flask-wtf = "*" Flask-Misaka = "*" flask-mwoauth = "*" WTForms = "*" -loginpass = {ref = 'ed9e527502bd19be6793f88ed71ca0e1764e9034',git = 'https://github.com/authlib/loginpass.git'} +loginpass = ">=0.4" requests = ">=2" raven = {extras = ['flask'],version = "*"} pykafka = ">=2" @@ -39,14 +39,16 @@ sickle = "*" python-snappy = "*" pymacaroons = "*" ftfy = "*" -citeproc-py = "*" -citeproc-py-styles = "*" +citeproc-py = ">=0.4" +citeproc-py-styles = ">=0.1" bs4 = "*" python-magic = "*" pylatexenc = "*" pygal = "*" elasticsearch-dsl = ">=6.0.0,<7.0.0" elasticsearch = ">=6.0.0,<7.0.0" +langcodes = ">=1.4" +dateparser = ">=0.7" [requires] # Python 3.5 is the bundled (system) version of python for Ubuntu 16.04 diff --git a/python/Pipfile.lock b/python/Pipfile.lock index 8ced4d88..b6e066b5 100644 --- a/python/Pipfile.lock +++ b/python/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7ef50f1c42fdcd59e1016ca4581bf6ab1fe60cd3133417532eddb107dd402c8d" + "sha256": "17edea9781496af462798b03388bcf0c86d3924a761168cb5d324dca5916ce46" }, "pipfile-spec": 6, "requires": { @@ -16,13 +16,20 @@ ] }, "default": { + "authlib": { + "hashes": [ + "sha256:401a4a184379a6551b79af30243ce1ad56da0a8da5b4925e0be260f07ee40009", + "sha256:e9a6a6c5748109bd9ce9f24e71d293156c43bd0d6595bc4e63dd8f5b9e809149" + ], + "version": "==0.13" + }, "beautifulsoup4": { "hashes": [ - "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", - "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", - "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", + "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", + "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" ], - "version": "==4.7.1" + "version": "==4.8.1" }, "blinker": { "hashes": [ @@ -39,43 +46,48 @@ }, "certifi": { "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.6.16" + "version": "==2019.11.28" }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" }, "chardet": { "hashes": [ @@ -109,35 +121,69 @@ }, "confluent-kafka": { "hashes": [ - "sha256:13d0146850c046b9e3dabbcb60bd7c4f02c7d4754b693266ee5bb0884faea2c7", - "sha256:1505de9c652f9b841ba600f35e6a7f4cae2e239e9b8255212433756d17c1aeed", - "sha256:1832373eee96b0ef246c773ec2613c382bf4577f0d42e2ce688e8d35ca373e69", - "sha256:3001d09e5efa357eb9f3166ef54703166d7b662eb55841b760a346313ca717ee", - "sha256:31b934aa821e4d6aff06dd260b14c03ca58a532bb160032acff573e2de0a4467", - "sha256:359d776fb0381147e4c9981676d3907f9dbd12733597a6fec455e5a516728ae2", - "sha256:472f63ee352a1464ff40325cc3a86aae2685716985bb2c9cad7e574f4203d664", - "sha256:53ec15b8b76109489c77c0fe3d2f5ff71398cef2db74053c4818f6667a14471d", - "sha256:6299420c462b274095d4624cf246dc90e49d6d3c1acd8b7222d39eb407476301", - "sha256:8a497aee87d01891090c9c59ee760387bc41c718a358dadc926a6cae36dfa4fa", - "sha256:8bd48e88b2d9ba42b58e8c5c1a266a4a7945757d1ae1c2bad3c0fbdf7b52c5e5", - "sha256:92684d50215c111025ebb40c813f695fd9df69f763f04e9b25fc4fd67e5c0d06", - "sha256:957417782592136c91c81d7161be8c6fc334f65d229a672d18c4cc85a0a09532", - "sha256:a41eda5c84e153e0ec8d854c064ebe242a8a6993f9069f6ae83dfd5f9dda1ac6", - "sha256:b4e5b7f5f597d4a40e0148e1fe509ab800231542ce7e8765265e8c251e6e3983", - "sha256:c41d12f4e1deb54e1fa6e40260d95aa7b1c1ebfa4bbddb8a03f06fde3611da71", - "sha256:c8b75e6e803e1a4346d12f212ead3871f86cf21e4d6b5028e8701f8e9c8f105b", - "sha256:c8dee478a46a9352224fce1d12756cff6cf50e2684121a118141c7908ae9ed3e", - "sha256:d072825c0f9dec85ebc5eaa887c324914cb3631c884b35fc3c173663d4222a3a", - "sha256:d328d11edfe049cbdd6bf0792bcff9a9afce26c25124f198fe0ea1f910b529b3", - "sha256:d52708474daec030399cf45a6bff55b13d631cdba6cc6546da351f675a901d44", - "sha256:dd02a60145961957ad293aed30a435de2069024aec134575bea937e50df211b3", - "sha256:eb900ef15b340a87185412b906cbef2450ab68907b84988a6af7fc8e1b017b6c", - "sha256:fa5d985e30fbcbccaf8b9b1e52137abfbbe0f913fc1baddb96b9476824013411", - "sha256:fb8ba7ddf85c80eb4d2e3c2f1b362793b9f5c78950bc1e93f60423241071fd9b", - "sha256:ff80ceca738f52d9058d146b7b063092a88281362606e6c917af2a9e60087a3d" + "sha256:0efd716da4f03f99d45fbb0d1583c5c8bf1eabc258a883588e3cd6ee06c0facb", + "sha256:1f52d6dd04fc81cb22e89a6b579cde83634f9c98218af6fcb794e8e924fc5de0", + "sha256:4025ccddbc79443a4e2342de0d770f669558eb737fca2e7851558cd45f78ef78", + "sha256:413cc9189ba7560bdc1010bbf3cabdcccfb794262f20aa867778e4059e789e88", + "sha256:4194c11fdfa47fe749eabe7f2bcd259e921a457b046df5fe82d11dd519412327", + "sha256:4d85e900ee59cba6acbe2ed415f6b8b86c86cca4f1cf0782bca24588cb594f52", + "sha256:6095b248c457518d3f83ecb981a819512241b3c3fd7f461f3c55405278ab7825", + "sha256:63245ea826238f7c41002c6583e44d07a3a25fbcedd320af4902bb22d17a67ec", + "sha256:70a2aa7056143ff30a8259d69327b38838f402fabbcc0699cfd8a7de4123825a", + "sha256:77f545e9b143b22d2f11609677aacf7d15855b0166a59634fb1b125ac209b7d7", + "sha256:81a1f91a82c65bc3d3d77dbb821b2c91e36ec482a5fc1dec8996c00c4c5bbd3b", + "sha256:8469a8780a8d6258d4be08dc882b1d2e8114cfb93ae3836c2841ecfc127efc03", + "sha256:910cd945e9684c4d1ee8777258e6f3fa6a54e5ff8d361bba65b03275d0e57f18", + "sha256:9350101ca2db756cb9ca2c5ef410a0c19b200267add97a76c2ffa167ac59685a", + "sha256:98258fbe75c5d67f0e6103eec1763f9bb713e4258dbdac45ecf8805a975fdd5c", + "sha256:99d75d06e79ec7d489cd410717d5ad303dee2a632db5d39f5d20f1ec1a40a23e", + "sha256:a1925564bb10556686a9857ce22c222f1d6e66079d3379e23edec0bb3c19703b", + "sha256:a4600737ed4ceb6c799319d1d2977c46c00966274d868bcdeba66b655558ee26", + "sha256:b96882b342b78b082aeedb56ee3c35d137adeadc26a63188ea36ae46ae1d485c", + "sha256:c57c58f812a21fa8ad9f8bac88b4cd026dfe5b4f1fc9353d4fc0186c9b6d558e", + "sha256:c72c9223a95b6668a805b175ab39e10a1cb2074e1cfa32a7cace758c744c4238", + "sha256:ce45cc45b0dd29fb5a297a7f248f98dc5bde0142d48e07a19140f45906296b14", + "sha256:e3bcb44595824dd1c78980bb57804a18ff8de16e257e747bd2724db88acfee25", + "sha256:f294a3c3115c2b12a8f0dec6165e1bd427f3a4ca6066a9028e13f9a516c9a162", + "sha256:f7a8bb7a5e80776a2dd8d4f4edbef1ebd5fdb2f443f87e95e08f6f7c598c33e1", + "sha256:fcb975bc7b26611e3c5c32f3c584ed860703f0496b22c096642ebc36d8cb4f12" ], "index": "pypi", - "version": "==1.1.0" + "version": "==1.3.0" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "dateparser": { + "hashes": [ + "sha256:983d84b5e3861cb0aa240cad07f12899bb10b62328aae188b9007e04ce37d665", + "sha256:e1eac8ef28de69a554d5fcdb60b172d526d61924b1a40afbbb08df459a36006b" + ], + "index": "pypi", + "version": "==0.7.2" }, "elasticsearch": { "hashes": [ @@ -160,11 +206,11 @@ }, "flask": { "hashes": [ - "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", - "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" ], "index": "pypi", - "version": "==1.0.3" + "version": "==1.1.1" }, "flask-debugtoolbar": { "hashes": [ @@ -183,6 +229,7 @@ }, "flask-misaka": { "hashes": [ + "sha256:bcfdacc0803ccea75d377737e82c83489b2153d922c9d9f9eabc5148d216ed70", "sha256:d0cfb0efd9e5afacda76defd4a605a68390f4fb1bef283c71534fd3ce0d3efb5", "sha256:f423c3beb5502742a57330a272f81d53223f6f99d45cc45b03926e3a3034f589" ], @@ -191,10 +238,10 @@ }, "flask-mwoauth": { "hashes": [ - "sha256:9ac2c802a2943798b8e72c13362adf1918f060f3deed3efd2c155e1ad680fccd" + "sha256:6df2e2448ea9251c61d4142da1aa2cd529e9ce22b440b577b1197912a8969be9" ], "index": "pypi", - "version": "==0.3.70" + "version": "==0.4.75" }, "flask-uuid": { "hashes": [ @@ -213,17 +260,16 @@ }, "ftfy": { "hashes": [ - "sha256:84a1614190173bb447ac9d581e50185c6aa35b538754b6bedaba0cc0f83d8e80", - "sha256:fa74757fb7cb444366fa6a79c2feabd40281a44dfbf6eaed492a804764ee26b2" + "sha256:6d7509c45e602dec890f0f6ee0623a8b5f50ec1188ac7ab9535e18e572c99bcc" ], "index": "pypi", - "version": "==5.5.1" + "version": "==5.6" }, "future": { "hashes": [ - "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], - "version": "==0.17.1" + "version": "==0.18.2" }, "idna": { "hashes": [ @@ -241,10 +287,10 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], - "version": "==2.10.1" + "version": "==2.10.3" }, "kazoo": { "hashes": [ @@ -253,38 +299,58 @@ ], "version": "==2.5.0" }, + "langcodes": { + "hashes": [ + "sha256:22cff103b6c0f3c9907d9a51c2d634177a50a189672ad16d959a3e2cd48eadab" + ], + "index": "pypi", + "version": "==1.4.1" + }, "loginpass": { - "git": "https://github.com/authlib/loginpass.git", - "ref": "ed9e527502bd19be6793f88ed71ca0e1764e9034" + "hashes": [ + "sha256:717c87c1870a7e00547fd9d989aea9b22232b2f48826f552d79c34a47f9618c9", + "sha256:9a6e6abef23f2fe9fa9c7999dbdf2f82d9be313e47fa71b7f234fd402acc091e" + ], + "index": "pypi", + "version": "==0.4" }, "lxml": { "hashes": [ - "sha256:06c7616601430aa140a69f97e3116308fffe0848f543b639a5ec2e8920ae72fd", - "sha256:177202792f9842374a8077735c69c41a4282183f7851443d2beb8ee310720819", - "sha256:19317ad721ceb9e39847d11131903931e2794e447d4751ebb0d9236f1b349ff2", - "sha256:36d206e62f3e5dbaafd4ec692b67157e271f5da7fd925fda8515da675eace50d", - "sha256:387115b066c797c85f9861a9613abf50046a15aac16759bc92d04f94acfad082", - "sha256:3ce1c49d4b4a7bc75fb12acb3a6247bb7a91fe420542e6d671ba9187d12a12c2", - "sha256:4d2a5a7d6b0dbb8c37dab66a8ce09a8761409c044017721c21718659fa3365a1", - "sha256:58d0a1b33364d1253a88d18df6c0b2676a1746d27c969dc9e32d143a3701dda5", - "sha256:62a651c618b846b88fdcae0533ec23f185bb322d6c1845733f3123e8980c1d1b", - "sha256:69ff21064e7debc9b1b1e2eee8c2d686d042d4257186d70b338206a80c5bc5ea", - "sha256:7060453eba9ba59d821625c6af6a266bd68277dce6577f754d1eb9116c094266", - "sha256:7d26b36a9c4bce53b9cfe42e67849ae3c5c23558bc08363e53ffd6d94f4ff4d2", - "sha256:83b427ad2bfa0b9705e02a83d8d607d2c2f01889eb138168e462a3a052c42368", - "sha256:923d03c84534078386cf50193057aae98fa94cace8ea7580b74754493fda73ad", - "sha256:b773715609649a1a180025213f67ffdeb5a4878c784293ada300ee95a1f3257b", - "sha256:baff149c174e9108d4a2fee192c496711be85534eab63adb122f93e70aa35431", - "sha256:bca9d118b1014b4c2d19319b10a3ebed508ff649396ce1855e1c96528d9b2fa9", - "sha256:ce580c28845581535dc6000fc7c35fdadf8bea7ccb57d6321b044508e9ba0685", - "sha256:d34923a569e70224d88e6682490e24c842907ba2c948c5fd26185413cbe0cd96", - "sha256:dd9f0e531a049d8b35ec5e6c68a37f1ba6ec3a591415e6804cbdf652793d15d7", - "sha256:ecb805cbfe9102f3fd3d2ef16dfe5ae9e2d7a7dfbba92f4ff1e16ac9784dbfb0", - "sha256:ede9aad2197a0202caff35d417b671f5f91a3631477441076082a17c94edd846", - "sha256:ef2d1fc370400e0aa755aab0b20cf4f1d0e934e7fd5244f3dd4869078e4942b9", - "sha256:f2fec194a49bfaef42a548ee657362af5c7a640da757f6f452a35da7dd9f923c" - ], - "version": "==4.3.4" + "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", + "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", + "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", + "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", + "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", + "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", + "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", + "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", + "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", + "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", + "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", + "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", + "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", + "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", + "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", + "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", + "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", + "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", + "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", + "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", + "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", + "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", + "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", + "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", + "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", + "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" + ], + "version": "==4.4.2" + }, + "marisa-trie": { + "hashes": [ + "sha256:4419abb6b603c97e863fad994abe57ed247fb12491f4bbacb2d762bd2e8958b6", + "sha256:c73bc25d868e8c4ea7aa7f1e19892db07bba2463351269b05340ccfa06eb2baf" + ], + "version": "==0.7.5" }, "markupsafe": { "hashes": [ @@ -327,17 +393,18 @@ }, "mwoauth": { "hashes": [ - "sha256:8a57a315732733240e9522d3c4e370cbdf2c045d00fe0dab433d6119fa09038f", - "sha256:ae0d094bd8817b690af5e8c52a113ae229e689fca59303aae0f0fdfb0ca911d1" + "sha256:9e0d70a1fa6f452584de1cb853ae6c11f41233549f7839cfb879f99410f6ad46", + "sha256:af4e9dcbf397255dc0a31dea7655a9b9e5de312fcfed816e80622437c5cf072d", + "sha256:ef3e86ab965848e71162990cb4e9c580408a098df793fc08511e062d12f068d4" ], - "version": "==0.3.3" + "version": "==0.3.7" }, "oauthlib": { "hashes": [ - "sha256:0ce32c5d989a1827e3f1148f98b9085ed2370fc939bf524c9c851d8714797298", - "sha256:3e1e14f6cde7e5475128d30e97edc3bfb4dc857cb884d8714ec161fdbb3b358e" + "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889", + "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea" ], - "version": "==3.0.1" + "version": "==3.1.0" }, "pycparser": { "hashes": [ @@ -362,6 +429,7 @@ }, "pykafka": { "hashes": [ + "sha256:6b075909a52cb0c95325bc16ab797bbcdbb37386652ea460705ed4472ce91459", "sha256:f0bbd394ae6970042a587c99fe4dc0966e67787249d963d4ce2f810dc9490577" ], "index": "pypi", @@ -369,10 +437,10 @@ }, "pylatexenc": { "hashes": [ - "sha256:ef2d5260c38e2cb4d2829e8b918914a558557820d4f57cb6588a81e827de2bb3" + "sha256:68f6de3921c54d09edb23f2e7625f047d3645cf4f341cc89153e466918411e99" ], "index": "pypi", - "version": "==1.5" + "version": "==2.1" }, "pymacaroons": { "hashes": [ @@ -393,6 +461,7 @@ "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:53126cd91356342dcae7e209f840212a58dcf1177ad52c1d938d428eebc9fee5", "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", @@ -401,6 +470,7 @@ "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:bf459128feb543cfca16a95f8da31e2e65e4c5257d2f3dfa8c0c1031139c9c92", "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" ], @@ -408,11 +478,11 @@ }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], "index": "pypi", - "version": "==2.8.0" + "version": "==2.8.1" }, "python-dotenv": { "hashes": [ @@ -438,10 +508,14 @@ "index": "pypi", "version": "==0.5.4" }, - "raven": { - "extras": [ - "flask" + "pytz": { + "hashes": [ + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" ], + "version": "==2019.3" + }, + "raven": { "hashes": [ "sha256:3fa6de6efa2493a7c827472e984ce9b020797d0da16f1db67197bcc23c8fae54", "sha256:44a13f87670836e153951af9a3c80405d36b43097db869a36e92809673692ce4" @@ -449,6 +523,32 @@ "index": "pypi", "version": "==6.10.0" }, + "regex": { + "hashes": [ + "sha256:0472acc4b6319801c1bc681d838c88ba1446f9ae199e01f6e41091c701fb3d42", + "sha256:16709434c4e2332ee8ba26ae339aceb8ab0b24b8398ebd0f52ebc943f45c4fc2", + "sha256:223fb63ec8dcab20b3318e93dcec4aee89e98b062934090bf29ffc374d2000a2", + "sha256:23c3ebf05d1cd3adb26723fd598e75724e0cdb7d6a35185ac0caf061cc6edb49", + "sha256:2404a50fb48badaf214b700f08822b68d93d79200e0aefd9569d0332d21fbfcb", + "sha256:2af3a7a16fed6eff85c25da106effa36f61cbbe801d00ade349b53ce7619eb15", + "sha256:37e018d3746baf159aedfc9773c3cafacbd10d354ba15484f5cfc8ed9da5748b", + "sha256:3c9c2988d02a9238a1975c70e87c6ce94e6f36dd8e372b66f468990cfe077434", + "sha256:47298bc8b89d1c747f0f5974aa528fc0b6b17396f1694136a224d51461279d83", + "sha256:4eeb0fe936797ae00a085f99802642bfc722b3b4ea557e9e7849cb621ea10c91", + "sha256:6881be0218b47ed76db033f252bab3f912dfe7ed1fe7baa9daebf51de08546a0", + "sha256:7ac08cee5055f548eed3889e9aaef15fd00172d037949496f1f0b34acb8a7c3e", + "sha256:7c5e2efcf079c35ff266c3f3a6708834f88f9fd04a3c16b855e036b2b7b1b543", + "sha256:8355eaa64724a0fdb010a1654b77cb3e375dc08b7f592cc4a1c05ac606aa481c", + "sha256:999a885f7f5194464238ad5d74b05982acee54002f3aa775d8e0e8c5fb74c06c", + "sha256:9fd2f4813eaa3e421e82819d38e5b634d900faff7ae5a80cd89ccff407175e69", + "sha256:a2e1e53df7dd27943da2b512895125b33fb20f81862c9fed7b3bab2a1de684d1", + "sha256:ab43bc0836820b7900dfffc025b996784aec26ec87dc1df4f95a40398760223f", + "sha256:ba449b56fa419fb19bf2a2438adbd2433f27087a6fe115917eaf9cfca684d5b6", + "sha256:d3f632cefad2cf247bd845794002585e3772288bfcb0dbac59fdecd32cd38b67", + "sha256:d51311496061863caae2cfe120cf1ef37900019b86c89c2d75f0918e0b4b8bf3" + ], + "version": "==2019.12.19" + }, "requests": { "hashes": [ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", @@ -459,10 +559,11 @@ }, "requests-oauthlib": { "hashes": [ - "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57", - "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140" + "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d", + "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a", + "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc" ], - "version": "==1.2.0" + "version": "==1.3.0" }, "sickle": { "hashes": [ @@ -474,30 +575,37 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "soupsieve": { "hashes": [ - "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece", - "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca" + "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5", + "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda" ], - "version": "==1.9.1" + "version": "==1.9.5" }, "tabulate": { "hashes": [ - "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943" + "sha256:5470cc6687a091c7042cee89b2946d9235fe9f6d49c193a4ae2ac7bf386737c8" + ], + "version": "==0.8.6" + }, + "tzlocal": { + "hashes": [ + "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048", + "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590" ], - "version": "==0.8.3" + "version": "==2.0.0" }, "urllib3": { "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.3" + "version": "==1.25.7" }, "wcwidth": { "hashes": [ @@ -508,10 +616,10 @@ }, "werkzeug": { "hashes": [ - "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", - "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" + "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", + "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" ], - "version": "==0.15.4" + "version": "==0.16.0" }, "wtforms": { "hashes": [ @@ -525,10 +633,10 @@ "develop": { "astroid": { "hashes": [ - "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" ], - "version": "==2.2.5" + "version": "==2.3.3" }, "atomicwrites": { "hashes": [ @@ -539,10 +647,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.1.0" + "version": "==19.3.0" }, "backcall": { "hashes": [ @@ -553,10 +661,10 @@ }, "certifi": { "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.6.16" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -567,46 +675,46 @@ }, "coverage": { "hashes": [ - "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", - "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", - "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", - "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", - "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", - "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", - "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", - "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", - "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", - "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", - "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", - "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", - "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", - "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", - "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", - "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", - "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", - "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", - "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", - "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", - "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", - "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", - "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", - "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", - "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", - "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", - "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", - "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", - "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", - "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", - "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" - ], - "version": "==4.5.3" + "sha256:0cd13a6e98c37b510a2d34c8281d5e1a226aaf9b65b7d770ef03c63169965351", + "sha256:1a4b6b6a2a3a6612e6361130c2cc3dc4378d8c221752b96167ccbad94b47f3cd", + "sha256:2ee55e6dba516ddf6f484aa83ccabbb0adf45a18892204c23486938d12258cde", + "sha256:3be5338a2eb4ef03c57f20917e1d12a1fd10e3853fed060b6d6b677cb3745898", + "sha256:44b783b02db03c4777d8cf71bae19eadc171a6f2a96777d916b2c30a1eb3d070", + "sha256:475bf7c4252af0a56e1abba9606f1e54127cdf122063095c75ab04f6f99cf45e", + "sha256:47c81ee687eafc2f1db7f03fbe99aab81330565ebc62fb3b61edfc2216a550c8", + "sha256:4a7f8e72b18f2aca288ff02255ce32cc830bc04d993efbc87abf6beddc9e56c0", + "sha256:50197163a22fd17f79086e087a787883b3ec9280a509807daf158dfc2a7ded02", + "sha256:56b13000acf891f700f5067512b804d1ec8c301d627486c678b903859d07f798", + "sha256:79388ae29c896299b3567965dbcd93255f175c17c6c7bca38614d12718c47466", + "sha256:79fd5d3d62238c4f583b75d48d53cdae759fe04d4fb18fe8b371d88ad2b6f8be", + "sha256:7fe3e2fde2bf1d7ce25ebcd2d3de3650b8d60d9a73ce6dcef36e20191291613d", + "sha256:81042a24f67b96e4287774014fa27220d8a4d91af1043389e4d73892efc89ac6", + "sha256:81326f1095c53111f8afc95da281e1414185f4a538609a77ca50bdfa39a6c207", + "sha256:8873dc0d8f42142ea9f20c27bbdc485190fff93823c6795be661703369e5877d", + "sha256:88d2cbcb0a112f47eef71eb95460b6995da18e6f8ca50c264585abc2c473154b", + "sha256:91f2491aeab9599956c45a77c5666d323efdec790bfe23fcceafcd91105d585a", + "sha256:979daa8655ae5a51e8e7a24e7d34e250ae8309fd9719490df92cbb2fe2b0422b", + "sha256:9c871b006c878a890c6e44a5b2f3c6291335324b298c904dc0402ee92ee1f0be", + "sha256:a6d092545e5af53e960465f652e00efbf5357adad177b2630d63978d85e46a72", + "sha256:b5ed7837b923d1d71c4f587ae1539ccd96bfd6be9788f507dbe94dab5febbb5d", + "sha256:ba259f68250f16d2444cbbfaddaa0bb20e1560a4fdaad50bece25c199e6af864", + "sha256:be1d89614c6b6c36d7578496dc8625123bda2ff44f224cf8b1c45b810ee7383f", + "sha256:c1b030a79749aa8d1f1486885040114ee56933b15ccfc90049ba266e4aa2139f", + "sha256:c95bb147fab76f2ecde332d972d8f4138b8f2daee6c466af4ff3b4f29bd4c19e", + "sha256:d52c1c2d7e856cecc05aa0526453cb14574f821b7f413cc279b9514750d795c1", + "sha256:d609a6d564ad3d327e9509846c2c47f170456344521462b469e5cb39e48ba31c", + "sha256:e1bad043c12fb58e8c7d92b3d7f2f49977dcb80a08a6d1e7a5114a11bf819fca", + "sha256:e5a675f6829c53c87d79117a8eb656cc4a5f8918185a32fc93ba09778e90f6db", + "sha256:fec32646b98baf4a22fdceb08703965bd16dea09051fbeb31a04b5b6e72b846c" + ], + "version": "==5.0" }, "decorator": { "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" ], - "version": "==4.4.0" + "version": "==4.4.1" }, "idna": { "hashes": [ @@ -617,18 +725,19 @@ }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" ], - "version": "==0.18" + "markers": "python_version < '3.8'", + "version": "==1.3.0" }, "ipython": { "hashes": [ - "sha256:54c5a8aa1eadd269ac210b96923688ccf01ebb2d0f21c18c3c717909583579a8", - "sha256:e840810029224b56cd0d9e7719dc3b39cf84d577f8ac686547c8ba7a06eeab26" + "sha256:007dcd929c14631f83daff35df0147ea51d1af420da303fd078343878bd5fb62", + "sha256:b0f2ef9eada4a68ef63ee10b6dde4f35c840035c50fd24265f8052c98947d5a4" ], "index": "pypi", - "version": "==7.5.0" + "version": "==6.5.0" }, "ipython-genutils": { "hashes": [ @@ -639,40 +748,43 @@ }, "isort": { "hashes": [ - "sha256:c40744b6bc5162bbb39c1257fe298b7a393861d50978b565f3ccd9cb9de0182a", - "sha256:f57abacd059dc3bd666258d1efb0377510a89777fda3e3274e3c01f7c03ae22d" + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" ], - "version": "==4.3.20" + "version": "==4.3.21" }, "jedi": { "hashes": [ - "sha256:49ccb782651bb6f7009810d17a3316f8867dde31654c750506970742e18b553d", - "sha256:79d0f6595f3846dffcbe667cc6dc821b96e5baa8add125176c31a3917eb19d58" + "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", + "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" ], - "version": "==0.14.0" + "version": "==0.15.1" }, "lazy-object-proxy": { "hashes": [ - "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", - "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", - "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", - "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", - "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", - "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", - "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", - "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", - "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", - "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", - "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", - "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", - "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", - "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", - "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", - "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", - "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", - "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" - ], - "version": "==1.4.1" + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" }, "mccabe": { "hashes": [ @@ -683,33 +795,33 @@ }, "more-itertools": { "hashes": [ - "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", - "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", + "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" ], "markers": "python_version > '2.7'", - "version": "==7.0.0" + "version": "==8.0.2" }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.0" + "version": "==19.2" }, "parso": { "hashes": [ - "sha256:5052bb33be034cba784193e74b1cde6ebf29ae8b8c1e4ad94df0c4209bfc4826", - "sha256:db5881df1643bf3e66c097bfd8935cf03eae73f4cb61ae4433c9ea4fb6613446" + "sha256:55cf25df1a35fd88b878715874d2c4dc1ad3f0eebd1e0266a67e1f55efccfbe1", + "sha256:5c1f7791de6bd5dbbeac8db0ef5594b36799de198b3f7f7014643b0c5536b9d3" ], - "version": "==0.5.0" + "version": "==0.5.2" }, "pathlib2": { "hashes": [ - "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", - "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db", + "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868" ], "markers": "python_version < '3.6'", - "version": "==2.3.3" + "version": "==2.3.5" }, "pexpect": { "hashes": [ @@ -735,34 +847,36 @@ }, "pluggy": { "hashes": [ - "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", - "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.12.0" + "version": "==0.13.1" }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:37925b37a4af1f6448c76b7606e0285f79f434ad246dda007a27411cca730c6d", + "sha256:dd4fca02c8069497ad931a2d09914c6b0d1b50151ce876bc15bde4c747090126", + "sha256:f7eec66105baf40eda9ab026cd8b2e251337eea8d111196695d82e0c5f0af852" ], - "version": "==2.0.9" + "version": "==1.0.18" }, "psycopg2": { "hashes": [ - "sha256:128d0fa910ada0157bba1cb74a9c5f92bb8a1dca77cf91a31eb274d1f889e001", - "sha256:227fd46cf9b7255f07687e5bde454d7d67ae39ca77e170097cdef8ebfc30c323", - "sha256:2315e7f104681d498ccf6fd70b0dba5bce65d60ac92171492bfe228e21dcc242", - "sha256:4b5417dcd2999db0f5a891d54717cfaee33acc64f4772c4bc574d4ff95ed9d80", - "sha256:640113ddc943522aaf71294e3f2d24013b0edd659b7820621492c9ebd3a2fb0b", - "sha256:897a6e838319b4bf648a574afb6cabcb17d0488f8c7195100d48d872419f4457", - "sha256:8dceca81409898c870e011c71179454962dec152a1a6b86a347f4be74b16d864", - "sha256:b1b8e41da09a0c3ef0b3d4bb72da0dde2abebe583c1e8462973233fd5ad0235f", - "sha256:cb407fccc12fc29dc331f2b934913405fa49b9b75af4f3a72d0f50f57ad2ca23", - "sha256:d3a27550a8185e53b244ad7e79e307594b92fede8617d80200a8cce1fba2c60f", - "sha256:f0e6b697a975d9d3ccd04135316c947dd82d841067c7800ccf622a8717e98df1" - ], - "version": "==2.8.3" + "sha256:4212ca404c4445dc5746c0d68db27d2cbfb87b523fe233dc84ecd24062e35677", + "sha256:47fc642bf6f427805daf52d6e52619fe0637648fe27017062d898f3bf891419d", + "sha256:72772181d9bad1fa349792a1e7384dde56742c14af2b9986013eb94a240f005b", + "sha256:8396be6e5ff844282d4d49b81631772f80dabae5658d432202faf101f5283b7c", + "sha256:893c11064b347b24ecdd277a094413e1954f8a4e8cdaf7ffbe7ca3db87c103f0", + "sha256:92a07dfd4d7c325dd177548c4134052d4842222833576c8391aab6f74038fc3f", + "sha256:965c4c93e33e6984d8031f74e51227bd755376a9df6993774fd5b6fb3288b1f4", + "sha256:9ab75e0b2820880ae24b7136c4d230383e07db014456a476d096591172569c38", + "sha256:b0845e3bdd4aa18dc2f9b6fb78fbd3d9d371ad167fd6d1b7ad01c0a6cdad4fc6", + "sha256:dca2d7203f0dfce8ea4b3efd668f8ea65cd2b35112638e488a4c12594015f67b", + "sha256:ed686e5926929887e2c7ae0a700e32c6129abb798b4ad2b846e933de21508151", + "sha256:ef6df7e14698e79c59c7ee7cf94cd62e5b869db369ed4b1b8f7b729ea825712a", + "sha256:f898e5cc0a662a9e12bde6f931263a1bbd350cfb18e1d5336a12927851825bb6" + ], + "version": "==2.8.4" }, "ptyprocess": { "hashes": [ @@ -780,41 +894,41 @@ }, "pygments": { "hashes": [ - "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", - "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.4.2" + "version": "==2.5.2" }, "pylint": { "hashes": [ - "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" ], "index": "pypi", - "version": "==2.3.1" + "version": "==2.4.4" }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" ], - "version": "==2.4.0" + "version": "==2.4.5" }, "pytest": { "hashes": [ - "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", - "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" + "sha256:65e92898fb5b61d0a1d7319c3e6dcf97e599e331cfdc2b27f20c0d87ece19239", + "sha256:9ea149066f566c943d3122f4b1cf1b577cab73189d11f490b54703fa5fa9df50" ], "index": "pypi", - "version": "==4.6.3" + "version": "==4.6.7" }, "pytest-cov": { "hashes": [ - "sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6", - "sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a" + "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", + "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626" ], "index": "pypi", - "version": "==2.7.1" + "version": "==2.8.1" }, "pytest-mock": { "hashes": [ @@ -826,12 +940,12 @@ }, "pytest-pylint": { "hashes": [ - "sha256:0dcb6cd429281f7493994e65324937bb295ef28f7632c046b45828380f4fe88a", - "sha256:45d2209b65989c8981289c8e75d0fde8c55e87cef27e97facb49f14773a8d3fb", - "sha256:7bfbb66fc6dc160193a9e813a7c55e5ae32028f18660deeb90e1cb7e980cbbac" + "sha256:8c38ea779e540e27ec4378b0820d906006e09f4ac834defbd886abbf57c7d2ec", + "sha256:a4f5e5007f88c2095dcac799e9f7eed3d7e7a2e657596e26093814980ff5fa20", + "sha256:a574c246535308f8f6ceac10fa82f8fffffa837071f7985b22515895185700c1" ], "index": "pypi", - "version": "==0.14.0" + "version": "==0.14.1" }, "pytest-pythonpath": { "hashes": [ @@ -850,53 +964,64 @@ }, "responses": { "hashes": [ - "sha256:502d9c0c8008439cfcdef7e251f507fcfdd503b56e8c0c87c3c3e3393953f790", - "sha256:97193c0183d63fba8cd3a041c75464e4b09ea0aff6328800d1546598567dde0b" + "sha256:caa5f7afd14666f970ea54a4125a639f6491218b45a013c6dc2544f0473ae2b8", + "sha256:f602986c715073b6bcb5d68a3225254aa447a1bd06040c66341816267d327721" ], "index": "pypi", - "version": "==0.10.6" + "version": "==0.10.8" + }, + "simplegeneric": { + "hashes": [ + "sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173" + ], + "version": "==0.8.1" }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "traitlets": { "hashes": [ - "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", - "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" ], - "version": "==4.3.2" + "version": "==4.3.3" }, "typed-ast": { "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" ], - "markers": "implementation_name == 'cpython'", + "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.0" }, "urllib3": { "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.3" + "version": "==1.25.7" }, "wcwidth": { "hashes": [ @@ -913,10 +1038,10 @@ }, "zipp": { "hashes": [ - "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", - "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.1" + "version": "==0.6.0" } } } diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index 13abb2e6..33f44600 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -8,6 +8,7 @@ import itertools import datetime import requests from confluent_kafka import Producer, KafkaException +from urllib.parse import urlparse, parse_qs from fatcat_tools.workers import most_recent_message from .harvest_common import HarvestState, requests_retry_session @@ -121,6 +122,10 @@ class HarvestCrossrefWorker: self.producer.poll(0) time.sleep(30.0) continue + if http_resp.status_code == 400: + print("skipping batch for {}, due to HTTP 400. Marking complete. Related: https://github.com/datacite/datacite/issues/897".format(date_str), + file=sys.stderr) + break http_resp.raise_for_status() resp = http_resp.json() items = self.extract_items(resp) @@ -179,7 +184,7 @@ class HarvestDataciteWorker(HarvestCrossrefWorker): """ def __init__(self, kafka_hosts, produce_topic, state_topic, contact_email, - api_host_url="https://api.datacite.org/works", + api_host_url="https://api.datacite.org/dois", start_date=None, end_date=None): super().__init__(kafka_hosts=kafka_hosts, produce_topic=produce_topic, @@ -193,11 +198,13 @@ class HarvestDataciteWorker(HarvestCrossrefWorker): self.name = "Datacite" def params(self, date_str): + """ + Dates have to be supplied in 2018-10-27T22:36:30.000Z format. + """ return { - 'from-update-date': date_str, - 'until-update-date': date_str, + 'query': 'updated:[{}T00:00:00.000Z TO {}T23:59:59.999Z]'.format(date_str, date_str), 'page[size]': self.api_batch_size, - 'page[number]': 1, + 'page[cursor]': 1, } def extract_items(self, resp): @@ -210,5 +217,20 @@ class HarvestDataciteWorker(HarvestCrossrefWorker): return obj['attributes']['doi'].encode('utf-8') def update_params(self, params, resp): - params['page[number]'] = resp['meta']['page'] + 1 + """ + Using cursor mechanism (https://support.datacite.org/docs/pagination#section-cursor). + + $ curl -sL https://is.gd/cLbE5h | jq -r .links.next + + Example: https://is.gd/cLbE5h + + Further API errors reported: + https://github.com/datacite/datacite/issues/897 (HTTP 400) + https://github.com/datacite/datacite/issues/898 (HTTP 500) + """ + parsed = urlparse(resp['links']['next']) + page_cursor = parse_qs(parsed.query).get('page[cursor]') + if not page_cursor: + raise ValueError('no page[cursor] in .links.next') + params['page[cursor]'] = page_cursor[0] return params diff --git a/python/fatcat_tools/importers/chocula.py b/python/fatcat_tools/importers/chocula.py index 6915ba98..eea50314 100644 --- a/python/fatcat_tools/importers/chocula.py +++ b/python/fatcat_tools/importers/chocula.py @@ -128,15 +128,15 @@ class ChoculaImporter(EntityImporter): existing.publisher = ce.publisher existing.container_type = existing.container_type or ce.container_type for k in ('urls', 'webarchive_urls'): - # update, or clobber/remove any existing values. often - # want/need to remove dead URLs + # update, which might clobber, but won't remove if ce.extra.get(k): existing.extra[k] = ce.extra.get(k, []) - elif k in existing.extra.keys(): - existing.extra.pop(k) + # note: in some cases we might *want* to clobber existing (if + # all URLs found to be bad), but being conservative for now so + # we don't clobber human edits for k in ('issne', 'issnp', 'country', 'sherpa_romeo', 'ezb', 'szczepanski', 'doaj'): - # update, but don't remove any existing value + # update/overwrite, but don't remove any existing value if ce.extra.get(k): existing.extra[k] = ce.extra[k] if ce.extra.get('languages'): diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py index c47f0aa7..33c40eff 100644 --- a/python/fatcat_tools/importers/ingest.py +++ b/python/fatcat_tools/importers/ingest.py @@ -29,6 +29,7 @@ class IngestFileResultImporter(EntityImporter): self.ingest_request_source_whitelist = [ 'fatcat-changelog', 'fatcat-ingest-container', + 'arabesque', ] if kwargs.get('skip_source_whitelist', False): self.ingest_request_source_whitelist = [] @@ -55,6 +56,10 @@ class IngestFileResultImporter(EntityImporter): if self.ingest_request_source_whitelist and source not in self.ingest_request_source_whitelist: self.counts['skip-ingest_request_source'] += 1 return False + if source.startswith('arabesque'): + if row['reqeust'].get('link_source') not in ('arxiv', 'pmc'): + self.counts['skip-arabesque-source'] += 1 + return False if source.startswith('savepapernow'): # never process async savepapernow requests self.counts['skip-savepapernow'] += 1 @@ -152,20 +157,22 @@ class IngestFileResultImporter(EntityImporter): if err.status != 404: raise err + # check for existing edits-in-progress with same file hash + for other in self._entity_queue: + if other.sha1 == fe.sha1: + self.counts['skip-in-queue'] += 1 + return False + if not existing: return True + # the following checks all assume there is an existing item + if (fe.release_ids[0] in existing.release_ids) and existing.urls: # TODO: could still, in theory update with the new URL? self.counts['exists'] += 1 return False - # check for existing edits-in-progress with same file hash - for other in self._entity_queue: - if other.sha1 == fe.sha1: - self.counts['skip-in-queue'] += 1 - return False - if not self.do_updates: self.counts['skip-update-disabled'] += 1 return False diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index 0a2c8610..554e052f 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -41,6 +41,10 @@ class OrcidImporter(EntityImporter): obj is a python dict (parsed from json). returns a CreatorEntity """ + + if not 'person' in obj: + return False + name = obj['person']['name'] if not name: return None diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py index 80cf986c..3611a299 100644 --- a/python/fatcat_tools/importers/pubmed.py +++ b/python/fatcat_tools/importers/pubmed.py @@ -8,6 +8,7 @@ from bs4 import BeautifulSoup from bs4.element import NavigableString import fatcat_openapi_client +from fatcat_tools.normal import * from .common import EntityImporter, clean, LANG_MAP_MARC # from: https://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.publication_types/?report=objectonly @@ -317,7 +318,7 @@ class PubmedImporter(EntityImporter): TODO: MEDLINE doesn't include PMC/OA license; could include in importer? """ - def __init__(self, api, issn_map_file, lookup_refs=False, **kwargs): + def __init__(self, api, issn_map_file, lookup_refs=True, **kwargs): eg_desc = kwargs.get('editgroup_description', "Automated import of PubMed/MEDLINE XML metadata") @@ -330,38 +331,9 @@ class PubmedImporter(EntityImporter): **kwargs) self.lookup_refs = lookup_refs - extid_map_file = kwargs.get('extid_map_file') - self.extid_map_db = None - if extid_map_file: - db_uri = "file:{}?mode=ro".format(extid_map_file) - print("Using external ID map: {}".format(db_uri)) - self.extid_map_db = sqlite3.connect(db_uri, uri=True) - else: - print("Not using external ID map") - self.create_containers = kwargs.get('create_containers', True) self.read_issn_map_file(issn_map_file) - def lookup_ext_ids(self, pmid): - if self.extid_map_db is None: - return dict(doi=None, core_id=None, pmid=None, pmcid=None, - wikidata_qid=None, arxiv_id=None, jstor_id=None) - row = self.extid_map_db.execute("SELECT core, doi, pmcid, wikidata FROM ids WHERE pmid=? LIMIT 1", - [pmid]).fetchone() - if row is None: - return dict(doi=None, core_id=None, pmid=None, pmcid=None, - wikidata_qid=None, arxiv_id=None, jstor_id=None) - row = [str(cell or '') or None for cell in row] - return dict( - core_id=row[0], - doi=row[1], - pmcid=row[2], - wikidata_qid=row[3], - # TODO: - arxiv_id=None, - jstor_id=None, - ) - def want(self, obj): return True @@ -376,20 +348,14 @@ class PubmedImporter(EntityImporter): identifiers = pubmed.ArticleIdList pmid = medline.PMID.string.strip() doi = identifiers.find("ArticleId", IdType="doi") - if doi: - doi = doi.string.lower().strip() - if doi.startswith('doi:'): - doi = doi[4:] - if not (doi.startswith('10.') and '/' in doi and doi.split('/')[1]) and len(doi.split()) == 1: - sys.stderr.write("BOGUS DOI: {}\n".format(doi)) - doi = None + if doi and doi.string: + doi = clean_doi(doi.string) + else: + doi = None pmcid = identifiers.find("ArticleId", IdType="pmc") if pmcid: - pmcid = pmcid.string.strip().upper() - # got a bunch of weird ones like "wst_2018_399" in the 2019 baseline - if not pmcid.startswith("PMC"): - pmcid = None + pmcid = clean_pmcid(pmcid.string.strip().upper()) release_type = None pub_types = [] @@ -486,6 +452,8 @@ class PubmedImporter(EntityImporter): pub_date = medline.Article.find('ArticleDate') if not pub_date: pub_date = journal.PubDate + if not pub_date: + pub_date = journal.JournalIssue.PubDate release_date = None release_year = None if pub_date.Year: @@ -498,8 +466,17 @@ class PubmedImporter(EntityImporter): int(pub_date.Day.string)) release_date = release_date.isoformat() except ValueError as ve: - sys.stderr.write("bad date, skipping: {}\n".format(ve)) + print("bad date, skipping: {}".format(ve), file=sys.stderr) release_date = None + elif pub_date.MedlineDate: + medline_date = pub_date.MedlineDate.string.strip() + if len(medline_date) >= 4 and medline_date[:4].isdigit(): + release_year = int(medline_date[:4]) + if release_year < 1300 or release_year > 2040: + print("bad medline year, skipping: {}".format(release_year), file=sys.stderr) + release_year = None + else: + print("unparsable medline date, skipping: {}".format(medline_date), file=sys.stderr) if journal.find("Title"): container_name = journal.Title.string @@ -641,19 +618,21 @@ class PubmedImporter(EntityImporter): if pubmed.ReferenceList: for ref in pubmed.ReferenceList.find_all('Reference'): ref_extra = dict() - ref_pmid = ref.find("ArticleId", IdType="pubmed") ref_doi = ref.find("ArticleId", IdType="doi") - ref_release_id = None + if ref_doi: + ref_doi = clean_doi(ref_doi.string) + ref_pmid = ref.find("ArticleId", IdType="pubmed") if ref_pmid: - ref_pmid = ref_pmid.string.strip() - ref_extra['pmid'] = ref_pmid - if self.lookup_refs: - ref_release_id = self.lookup_pmid(ref_pmid) + ref_pmid = clean_pmid(ref_pmid.string) + ref_release_id = None if ref_doi: - ref_doi = ref_doi.string.lower().strip() ref_extra['doi'] = ref_doi if self.lookup_refs: ref_release_id = self.lookup_doi(ref_doi) + if ref_pmid: + ref_extra['pmid'] = ref_pmid + if self.lookup_refs: + ref_release_id = self.lookup_pmid(ref_pmid) ref_raw = ref.Citation if ref_raw: ref_extra['unstructured'] = ref_raw.string @@ -668,7 +647,6 @@ class PubmedImporter(EntityImporter): # extra: # translation_of - # subtitle # aliases # container_name # group-title @@ -729,8 +707,9 @@ class PubmedImporter(EntityImporter): if err.status != 404: raise err if existing and existing.ext_ids.pmid and existing.ext_ids.pmid != re.ext_ids.pmid: - warnings.warn("PMID/DOI mismatch: release {}, pmid {} != {}".format( - existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid)) + warn_str = "PMID/DOI mismatch: release {}, pmid {} != {}".format( + existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid) + warnings.warn(warn_str) self.counts['warn-pmid-doi-mismatch'] += 1 # don't clobber DOI, but do group together re.ext_ids.doi = None @@ -748,6 +727,15 @@ class PubmedImporter(EntityImporter): existing.ext_ids.pmcid = existing.ext_ids.pmcid or re.ext_ids.pmcid existing.refs = existing.refs or re.refs existing.extra['pubmed'] = re.extra['pubmed'] + # update subtitle in-place first + if not existing.subtitle and existing.extra.get('subtitle'): + subtitle = existing.extra.pop('subtitle') + if type(subtitle) == list: + subtitle = subtitle[0] + if subtitle: + existing.subtitle = subtitle + if not existing.subtitle: + existing.subtitle = re.subtitle try: self.api.update_release(self.get_editgroup_id(), existing.ident, existing) self.counts['update'] += 1 diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py index 80bcfa5a..a77c5eb0 100644 --- a/python/fatcat_tools/normal.py +++ b/python/fatcat_tools/normal.py @@ -19,7 +19,10 @@ def clean_doi(raw): Returns None if not a valid DOI """ + if not raw: + return None raw = raw.strip() + raw = raw.replace('\u2013', '-') # emdash if len(raw.split()) != 1: return None if raw.startswith("doi:"): @@ -32,6 +35,8 @@ def clean_doi(raw): raw = raw[8:] if raw.startswith("dx.doi.org/"): raw = raw[11:] + if raw[7:9] == "//": + raw = raw[:8] + raw[9:] if not raw.startswith("10."): return None if not DOI_REGEX.fullmatch(raw): @@ -40,6 +45,10 @@ def clean_doi(raw): def test_clean_doi(): assert clean_doi("10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.23750/abm.v88i2 -s.6506") == None + assert clean_doi("10.17167/mksz.2017.2.129–155") == "10.17167/mksz.2017.2.129-155" assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf" @@ -54,6 +63,8 @@ def clean_arxiv_id(raw): Works with versioned or un-versioned arxiv identifiers. """ + if not raw: + return None raw = raw.strip() if raw.lower().startswith("arxiv:"): raw = raw[6:] @@ -90,7 +101,26 @@ def test_clean_arxiv_id(): assert clean_arxiv_id("0806.v1") == None assert clean_arxiv_id("08062878v1") == None +def clean_pmid(raw): + if not raw: + return None + raw = raw.strip() + if len(raw.split()) != 1: + return None + if raw.isdigit(): + return raw + return None + +def test_clean_pmid(): + assert clean_pmid("1234") == "1234" + assert clean_pmid("1234 ") == "1234" + assert clean_pmid("PMC123") == None + assert clean_sha1("qfba3") == None + assert clean_sha1("") == None + def clean_pmcid(raw): + if not raw: + return None raw = raw.strip() if len(raw.split()) != 1: return None @@ -99,6 +129,8 @@ def clean_pmcid(raw): return None def clean_sha1(raw): + if not raw: + return None raw = raw.strip().lower() if len(raw.split()) != 1: return None @@ -134,6 +166,8 @@ def test_clean_sha256(): ISSN_REGEX = re.compile("^\d{4}-\d{3}[0-9X]$") def clean_issn(raw): + if not raw: + return None raw = raw.strip().upper() if len(raw) != 9: return None @@ -150,6 +184,8 @@ def test_clean_issn(): ISBN13_REGEX = re.compile("^97(?:8|9)-\d{1,5}-\d{1,7}-\d{1,6}-\d$") def clean_isbn13(raw): + if not raw: + return None raw = raw.strip() if not ISBN13_REGEX.fullmatch(raw): return None @@ -164,6 +200,8 @@ def test_clean_isbn13(): ORCID_REGEX = re.compile("^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$") def clean_orcid(raw): + if not raw: + return None raw = raw.strip() if not ORCID_REGEX.fullmatch(raw): return None diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py index e08d56b8..d6393753 100644 --- a/python/fatcat_tools/transforms/ingest.py +++ b/python/fatcat_tools/transforms/ingest.py @@ -35,12 +35,12 @@ def release_ingest_request(release, oa_only=False, ingest_request_source='fatcat url = "https://doi.org/{}".format(release.ext_ids.doi) link_source = "doi" link_source_id = release.ext_ids.doi - elif release.ext_ids.pmcid and release.ext_ids.pmid: + elif release.ext_ids.pmcid: # TODO: how to tell if an author manuscript in PMC vs. published? #url = "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/".format(release.ext_ids.pmcid) url = "http://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.ext_ids.pmcid) - link_source = "pubmed" - link_source_id = release.ext_ids.pmid + link_source = "pmc" + link_source_id = release.ext_ids.pmcid if not url: return None @@ -48,7 +48,7 @@ def release_ingest_request(release, oa_only=False, ingest_request_source='fatcat ext_ids = release.ext_ids.to_dict() ext_ids = dict([(k, v) for (k, v) in ext_ids.items() if v]) - if oa_only and link_source not in ('arxiv', 'pubmed'): + if oa_only and link_source not in ('arxiv', 'pmc'): es = release_to_elasticsearch(release) if not es['is_oa']: return None diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py index 5539cc20..dd322cff 100644 --- a/python/fatcat_web/forms.py +++ b/python/fatcat_web/forms.py @@ -394,6 +394,8 @@ class SavePaperNowForm(FlaskForm): ingest_request = { 'ingest_type': self.ingest_type.data, 'ingest_request_source': ingest_request_source, + 'link_source': 'spn', + 'link_source_id': release.ident, 'base_url': base_url, 'fatcat': { 'release_ident': release.ident, diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 390327bc..df26a738 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -65,7 +65,7 @@ <table class="ui definition single line fixed compact small unstackable table"> <tbody> {% for (key, value) in extra.items()|sort %} - {% if key in ("ia", "crossref", "kbart", "arxiv", "jstor", "pubmed") and value is mapping and value %} + {% if key in ("ia", "crossref", "datacite", "kbart", "arxiv", "jstor", "pubmed") and value is mapping and value %} {% for (inner_key, inner_value) in value.items()|sort %} <tr><td class="three wide right aligned"><code>{{ key }}.{{ inner_key }}</code></td> <td class="seven wide"><code>{{ inner_value }}</code> @@ -273,4 +273,4 @@ found.count_returned }} out of {{ found.count_found }} results</i> </div> -{%- endmacro %}
\ No newline at end of file +{%- endmacro %} diff --git a/python/fatcat_web/templates/release_view.html b/python/fatcat_web/templates/release_view.html index b4e0ba25..83ecd1c8 100644 --- a/python/fatcat_web/templates/release_view.html +++ b/python/fatcat_web/templates/release_view.html @@ -405,11 +405,15 @@ <div class="ui segment attached accordion"> <div class="title" style="padding: 0px;"><i class="dropdown icon"></i><b>Lookup Links</b></div> <div class="content"> - {% if release.container != None and release.container.issnl != None %} - <a href="http://www.sherpa.ac.uk/romeo/issn/{{ release.container.issnl }}/">SHERPA/RoMEO</a> (journal policies)<br/> - {% endif %} {% if release != None and release.ext_ids.doi != None %} - <a href="https://oadoi.org/{{ release.ext_ids.doi }}">oaDOI/unpaywall</a><br/> + <a href="https://oadoi.org/{{ release.ext_ids.doi }}">oaDOI/unpaywall</a> (OA fulltext)<br/> + {% endif %} + {% if release.ext_ids.doi %} + {% if release.extra.datacite %} + <a href="https://api.datacite.org/dois/{{ release.ext_ids.doi }}">Datacite Metadata</a> (via API)<br> + {% else %} + <a href="https://api.crossref.org/v1/works/http://dx.doi.org/{{ release.ext_ids.doi }}">Crossref Metadata</a> (via API)<br> + {% endif %} {% endif %} {% if release.ext_ids.isbn13 != None %} <a href="https://openlibrary.org/search?isbn={{ release.ext_ids.isbn13 }}">Open Library</a><br> @@ -417,12 +421,12 @@ {% else %} <a href="http://www.worldcat.org/search?qt=worldcat_org_art&q={{ release.title|urlencode }}">Worldcat</a><br> {% endif %} - {% if release.ext_ids.doi %} - <a href="https://api.crossref.org/v1/works/http://dx.doi.org/{{ release.ext_ids.doi }}">Crossref Metadata</a> (via API)<br> + {% if release.container != None and release.container.issnl != None %} + <a href="http://www.sherpa.ac.uk/romeo/issn/{{ release.container.issnl }}/">SHERPA/RoMEO</a> (journal policies)<br/> {% endif %} <a href="https://www.wikidata.org/w/index.php?search={{ release.title|urlencode }}">wikidata.org</a><br> <a href="https://core.ac.uk/search?q={{ release.title|urlencode }}">CORE.ac.uk</a><br> - <a href="https://www.semanticscholar.org/search?q={{ release.title|urlencode }}">Semantic Scholar</a> (CS, neuro)<br> + <a href="https://www.semanticscholar.org/search?q={{ release.title|urlencode }}">Semantic Scholar</a><br> <a href="https://scholar.google.com/scholar?q={{ release.title|urlencode }}">Google Scholar</a><br> </div> </div> diff --git a/python/tests/files/datacite_api.json b/python/tests/files/datacite_api.json new file mode 100644 index 00000000..7852d730 --- /dev/null +++ b/python/tests/files/datacite_api.json @@ -0,0 +1 @@ +{"data":[{"id":"10.3204/desy-2014-00864","type":"dois","attributes":{"doi":"10.3204/desy-2014-00864","identifiers":[{"identifier":"https://doi.org/10.3204/desy-2014-00864","identifierType":"DOI"}],"creators":[{"name":"Abdo, A. A.","nameType":"Personal","givenName":"A. A.","familyName":"Abdo","affiliation":[]},{"name":"Ajello, M.","nameType":"Personal","givenName":"M.","familyName":"Ajello","affiliation":[]},{"name":"Allafort, A.","nameType":"Personal","givenName":"A.","familyName":"Allafort","affiliation":[]},{"name":"Baldini, L.","nameType":"Personal","givenName":"L.","familyName":"Baldini","affiliation":[]},{"name":"Ballet, J.","nameType":"Personal","givenName":"J.","familyName":"Ballet","affiliation":[]},{"name":"Barbiellini, G.","nameType":"Personal","givenName":"G.","familyName":"Barbiellini","affiliation":[]},{"name":"Baring, M. G.","nameType":"Personal","givenName":"M. G.","familyName":"Baring","affiliation":[]},{"name":"Bastieri, D.","nameType":"Personal","givenName":"D.","familyName":"Bastieri","affiliation":[]},{"name":"Belfiore, A.","nameType":"Personal","givenName":"A.","familyName":"Belfiore","affiliation":[]},{"name":"Bellazzini, R.","nameType":"Personal","givenName":"R.","familyName":"Bellazzini","affiliation":[]},{"name":"Bhattacharyya, B.","nameType":"Personal","givenName":"B.","familyName":"Bhattacharyya","affiliation":[]},{"name":"Bissaldi, E.","nameType":"Personal","givenName":"E.","familyName":"Bissaldi","affiliation":[]},{"name":"Bloom, E. D.","nameType":"Personal","givenName":"E. D.","familyName":"Bloom","affiliation":[]},{"name":"Bonamente, E.","nameType":"Personal","givenName":"E.","familyName":"Bonamente","affiliation":[]},{"name":"Bottacini, E.","nameType":"Personal","givenName":"E.","familyName":"Bottacini","affiliation":[]},{"name":"Brandt, T. J.","nameType":"Personal","givenName":"T. J.","familyName":"Brandt","affiliation":[]},{"name":"Bregeon, J.","nameType":"Personal","givenName":"J.","familyName":"Bregeon","affiliation":[]},{"name":"Brigida, M.","nameType":"Personal","givenName":"M.","familyName":"Brigida","affiliation":[]},{"name":"Bruel, P.","nameType":"Personal","givenName":"P.","familyName":"Bruel","affiliation":[]},{"name":"Buehler, R.","nameType":"Personal","givenName":"R.","familyName":"Buehler","affiliation":["Deutsches Elektronen-Synchrotron"]},{"name":"Burgay, M.","nameType":"Personal","givenName":"M.","familyName":"Burgay","affiliation":[]},{"name":"Burnett, T. H.","nameType":"Personal","givenName":"T. H.","familyName":"Burnett","affiliation":[]},{"name":"Busetto, G.","nameType":"Personal","givenName":"G.","familyName":"Busetto","affiliation":[]},{"name":"Buson, S.","nameType":"Personal","givenName":"S.","familyName":"Buson","affiliation":[]},{"name":"Caliandro, G. A.","nameType":"Personal","givenName":"G. A.","familyName":"Caliandro","affiliation":[]},{"name":"Cameron, R. A.","nameType":"Personal","givenName":"R. A.","familyName":"Cameron","affiliation":[]},{"name":"Camilo, F.","nameType":"Personal","givenName":"F.","familyName":"Camilo","affiliation":[]},{"name":"Caraveo, P. A.","nameType":"Personal","givenName":"P. A.","familyName":"Caraveo","affiliation":[]},{"name":"Casandjian, J. M.","nameType":"Personal","givenName":"J. M.","familyName":"Casandjian","affiliation":[]},{"name":"Cecchi, C.","nameType":"Personal","givenName":"C.","familyName":"Cecchi","affiliation":[]},{"name":"Çelik, Ö.","nameType":"Personal","givenName":"Ö.","familyName":"Çelik","affiliation":[]},{"name":"Charles, E.","nameType":"Personal","givenName":"E.","familyName":"Charles","affiliation":[]},{"name":"Chaty, S.","nameType":"Personal","givenName":"S.","familyName":"Chaty","affiliation":[]},{"name":"Chaves, R. C. G.","nameType":"Personal","givenName":"R. C. G.","familyName":"Chaves","affiliation":[]},{"name":"Chekhtman, A.","nameType":"Personal","givenName":"A.","familyName":"Chekhtman","affiliation":[]},{"name":"Chen, A. W.","nameType":"Personal","givenName":"A. W.","familyName":"Chen","affiliation":[]},{"name":"Chiang, J.","nameType":"Personal","givenName":"J.","familyName":"Chiang","affiliation":[]},{"name":"Chiaro, G.","nameType":"Personal","givenName":"G.","familyName":"Chiaro","affiliation":[]},{"name":"Ciprini, S.","nameType":"Personal","givenName":"S.","familyName":"Ciprini","affiliation":[]},{"name":"Claus, R.","nameType":"Personal","givenName":"R.","familyName":"Claus","affiliation":[]},{"name":"Cognard, I.","nameType":"Personal","givenName":"I.","familyName":"Cognard","affiliation":[]},{"name":"Cohen-Tanugi, J.","nameType":"Personal","givenName":"J.","familyName":"Cohen-Tanugi","affiliation":[]},{"name":"Cominsky, L. R.","nameType":"Personal","givenName":"L. R.","familyName":"Cominsky","affiliation":[]},{"name":"Conrad, J.","nameType":"Personal","givenName":"J.","familyName":"Conrad","affiliation":[]},{"name":"Cutini, S.","nameType":"Personal","givenName":"S.","familyName":"Cutini","affiliation":[]},{"name":"D'Ammando, F.","nameType":"Personal","givenName":"F.","familyName":"D'Ammando","affiliation":[]},{"name":"De Angelis, A.","nameType":"Personal","givenName":"A.","familyName":"De Angelis","affiliation":[]},{"name":"DeCesar, M. E.","nameType":"Personal","givenName":"M. E.","familyName":"DeCesar","affiliation":[]},{"name":"De Luca, A.","nameType":"Personal","givenName":"A.","familyName":"De Luca","affiliation":[]},{"name":"Den Hartog, P. R.","nameType":"Personal","givenName":"P. R.","familyName":"Den Hartog","affiliation":[]},{"name":"De Palma, F.","nameType":"Personal","givenName":"F.","familyName":"De Palma","affiliation":[]},{"name":"Dermer, C. D.","nameType":"Personal","givenName":"C. D.","familyName":"Dermer","affiliation":[]},{"name":"Desvignes, G.","nameType":"Personal","givenName":"G.","familyName":"Desvignes","affiliation":[]},{"name":"Digel, S. W.","nameType":"Personal","givenName":"S. W.","familyName":"Digel","affiliation":[]},{"name":"Di Venere, L.","nameType":"Personal","givenName":"L.","familyName":"Di Venere","affiliation":[]},{"name":"Drell, P. S.","nameType":"Personal","givenName":"P. S.","familyName":"Drell","affiliation":[]},{"name":"Drlica-Wagner, A.","nameType":"Personal","givenName":"A.","familyName":"Drlica-Wagner","affiliation":[]},{"name":"Dubois, R.","nameType":"Personal","givenName":"R.","familyName":"Dubois","affiliation":[]},{"name":"Dumora, D.","nameType":"Personal","givenName":"D.","familyName":"Dumora","affiliation":[]},{"name":"Espinoza, C. M.","nameType":"Personal","givenName":"C. M.","familyName":"Espinoza","affiliation":[]},{"name":"Falletti, L.","nameType":"Personal","givenName":"L.","familyName":"Falletti","affiliation":[]},{"name":"Favuzzi, C.","nameType":"Personal","givenName":"C.","familyName":"Favuzzi","affiliation":[]},{"name":"Ferrara, E. C.","nameType":"Personal","givenName":"E. C.","familyName":"Ferrara","affiliation":[]},{"name":"Focke, W. B.","nameType":"Personal","givenName":"W. B.","familyName":"Focke","affiliation":[]},{"name":"Franckowiak, A.","nameType":"Personal","givenName":"A.","familyName":"Franckowiak","affiliation":[]},{"name":"Freire, P. C. C.","nameType":"Personal","givenName":"P. C. C.","familyName":"Freire","affiliation":[]},{"name":"Funk, S.","nameType":"Personal","givenName":"S.","familyName":"Funk","affiliation":[]},{"name":"Fusco, P.","nameType":"Personal","givenName":"P.","familyName":"Fusco","affiliation":[]},{"name":"Gargano, F.","nameType":"Personal","givenName":"F.","familyName":"Gargano","affiliation":[]},{"name":"Gasparrini, D.","nameType":"Personal","givenName":"D.","familyName":"Gasparrini","affiliation":[]},{"name":"Germani, S.","nameType":"Personal","givenName":"S.","familyName":"Germani","affiliation":[]},{"name":"Giglietto, N.","nameType":"Personal","givenName":"N.","familyName":"Giglietto","affiliation":[]},{"name":"Giommi, P.","nameType":"Personal","givenName":"P.","familyName":"Giommi","affiliation":[]},{"name":"Giordano, F.","nameType":"Personal","givenName":"F.","familyName":"Giordano","affiliation":[]},{"name":"Giroletti, M.","nameType":"Personal","givenName":"M.","familyName":"Giroletti","affiliation":[]},{"name":"Glanzman, T.","nameType":"Personal","givenName":"T.","familyName":"Glanzman","affiliation":[]},{"name":"Godfrey, G.","nameType":"Personal","givenName":"G.","familyName":"Godfrey","affiliation":[]},{"name":"Gotthelf, E. V.","nameType":"Personal","givenName":"E. V.","familyName":"Gotthelf","affiliation":[]},{"name":"Grenier, I. A.","nameType":"Personal","givenName":"I. A.","familyName":"Grenier","affiliation":[]},{"name":"Grondin, M.-H.","nameType":"Personal","givenName":"M.-H.","familyName":"Grondin","affiliation":[]},{"name":"Grove, J. E.","nameType":"Personal","givenName":"J. E.","familyName":"Grove","affiliation":[]},{"name":"Guillemot, L.","nameType":"Personal","givenName":"L.","familyName":"Guillemot","affiliation":[]},{"name":"Guiriec, S.","nameType":"Personal","givenName":"S.","familyName":"Guiriec","affiliation":[]},{"name":"Hadasch, D.","nameType":"Personal","givenName":"D.","familyName":"Hadasch","affiliation":[]},{"name":"Hanabata, Y.","nameType":"Personal","givenName":"Y.","familyName":"Hanabata","affiliation":[]},{"name":"Harding, A. K.","nameType":"Personal","givenName":"A. K.","familyName":"Harding","affiliation":[]},{"name":"Hayashida, M.","nameType":"Personal","givenName":"M.","familyName":"Hayashida","affiliation":[]},{"name":"Hays, E.","nameType":"Personal","givenName":"E.","familyName":"Hays","affiliation":[]},{"name":"Hessels, J.","nameType":"Personal","givenName":"J.","familyName":"Hessels","affiliation":[]},{"name":"Hewitt, J.","nameType":"Personal","givenName":"J.","familyName":"Hewitt","affiliation":[]},{"name":"Hill, A. B.","nameType":"Personal","givenName":"A. B.","familyName":"Hill","affiliation":[]},{"name":"Horan, D.","nameType":"Personal","givenName":"D.","familyName":"Horan","affiliation":[]},{"name":"Hou, X.","nameType":"Personal","givenName":"X.","familyName":"Hou","affiliation":[]},{"name":"Hughes, R. E.","nameType":"Personal","givenName":"R. E.","familyName":"Hughes","affiliation":[]},{"name":"Jackson, M. S.","nameType":"Personal","givenName":"M. S.","familyName":"Jackson","affiliation":[]},{"name":"Janssen, G. H.","nameType":"Personal","givenName":"G. H.","familyName":"Janssen","affiliation":[]},{"name":"Jogler, T.","nameType":"Personal","givenName":"T.","familyName":"Jogler","affiliation":[]},{"name":"Jóhannesson, G.","nameType":"Personal","givenName":"G.","familyName":"Jóhannesson","affiliation":[]},{"name":"Johnson, R. P.","nameType":"Personal","givenName":"R. P.","familyName":"Johnson","affiliation":[]},{"name":"Johnson, A. S.","nameType":"Personal","givenName":"A. S.","familyName":"Johnson","affiliation":[]},{"name":"Johnson, T. J.","nameType":"Personal","givenName":"T. J.","familyName":"Johnson","affiliation":[]},{"name":"Johnson, W. N.","nameType":"Personal","givenName":"W. N.","familyName":"Johnson","affiliation":[]},{"name":"Johnston, S.","nameType":"Personal","givenName":"S.","familyName":"Johnston","affiliation":[]},{"name":"Kamae, T.","nameType":"Personal","givenName":"T.","familyName":"Kamae","affiliation":[]},{"name":"Kataoka, J.","nameType":"Personal","givenName":"J.","familyName":"Kataoka","affiliation":[]},{"name":"Keith, M.","nameType":"Personal","givenName":"M.","familyName":"Keith","affiliation":[]},{"name":"Kerr, M.","nameType":"Personal","givenName":"M.","familyName":"Kerr","affiliation":[]},{"name":"Knödlseder, J.","nameType":"Personal","givenName":"J.","familyName":"Knödlseder","affiliation":[]},{"name":"Kramer, M.","nameType":"Personal","givenName":"M.","familyName":"Kramer","affiliation":[]},{"name":"Kuss, M.","nameType":"Personal","givenName":"M.","familyName":"Kuss","affiliation":[]},{"name":"Lande, J.","nameType":"Personal","givenName":"J.","familyName":"Lande","affiliation":[]},{"name":"Larsson, S.","nameType":"Personal","givenName":"S.","familyName":"Larsson","affiliation":[]},{"name":"Latronico, L.","nameType":"Personal","givenName":"L.","familyName":"Latronico","affiliation":[]},{"name":"Lemoine-Goumard, M.","nameType":"Personal","givenName":"M.","familyName":"Lemoine-Goumard","affiliation":[]},{"name":"Longo, F.","nameType":"Personal","givenName":"F.","familyName":"Longo","affiliation":[]},{"name":"Loparco, F.","nameType":"Personal","givenName":"F.","familyName":"Loparco","affiliation":[]},{"name":"Lovellette, M. N.","nameType":"Personal","givenName":"M. N.","familyName":"Lovellette","affiliation":[]},{"name":"Lubrano, P.","nameType":"Personal","givenName":"P.","familyName":"Lubrano","affiliation":[]},{"name":"Lyne, A. G.","nameType":"Personal","givenName":"A. G.","familyName":"Lyne","affiliation":[]},{"name":"Manchester, R. N.","nameType":"Personal","givenName":"R. N.","familyName":"Manchester","affiliation":[]},{"name":"Marelli, M.","nameType":"Personal","givenName":"M.","familyName":"Marelli","affiliation":[]},{"name":"Massaro, F.","nameType":"Personal","givenName":"F.","familyName":"Massaro","affiliation":[]},{"name":"Mayer, Michael","nameType":"Personal","givenName":"Michael","familyName":"Mayer","affiliation":["Deutsches Elektronen-Synchrotron"]},{"name":"Mazziotta, M. N.","nameType":"Personal","givenName":"M. N.","familyName":"Mazziotta","affiliation":[]},{"name":"McEnery, J. E.","nameType":"Personal","givenName":"J. E.","familyName":"McEnery","affiliation":[]},{"name":"McLaughlin, M. A.","nameType":"Personal","givenName":"M. A.","familyName":"McLaughlin","affiliation":[]},{"name":"Mehault, J.","nameType":"Personal","givenName":"J.","familyName":"Mehault","affiliation":[]},{"name":"Michelson, P. F.","nameType":"Personal","givenName":"P. F.","familyName":"Michelson","affiliation":[]},{"name":"Mignani, R. P.","nameType":"Personal","givenName":"R. P.","familyName":"Mignani","affiliation":[]},{"name":"Mitthumsiri, W.","nameType":"Personal","givenName":"W.","familyName":"Mitthumsiri","affiliation":[]},{"name":"Mizuno, T.","nameType":"Personal","givenName":"T.","familyName":"Mizuno","affiliation":[]},{"name":"Moiseev, A. A.","nameType":"Personal","givenName":"A. A.","familyName":"Moiseev","affiliation":[]},{"name":"Monzani, M. E.","nameType":"Personal","givenName":"M. E.","familyName":"Monzani","affiliation":[]},{"name":"Morselli, A.","nameType":"Personal","givenName":"A.","familyName":"Morselli","affiliation":[]},{"name":"Moskalenko, I. V.","nameType":"Personal","givenName":"I. V.","familyName":"Moskalenko","affiliation":[]},{"name":"Murgia, S.","nameType":"Personal","givenName":"S.","familyName":"Murgia","affiliation":[]},{"name":"Nakamori, T.","nameType":"Personal","givenName":"T.","familyName":"Nakamori","affiliation":[]},{"name":"Nemmen, R.","nameType":"Personal","givenName":"R.","familyName":"Nemmen","affiliation":[]},{"name":"Nuss, E.","nameType":"Personal","givenName":"E.","familyName":"Nuss","affiliation":[]},{"name":"Ohno, M.","nameType":"Personal","givenName":"M.","familyName":"Ohno","affiliation":[]},{"name":"Ohsugi, T.","nameType":"Personal","givenName":"T.","familyName":"Ohsugi","affiliation":[]},{"name":"Orienti, M.","nameType":"Personal","givenName":"M.","familyName":"Orienti","affiliation":[]},{"name":"Orlando, E.","nameType":"Personal","givenName":"E.","familyName":"Orlando","affiliation":[]},{"name":"Ormes, J. F.","nameType":"Personal","givenName":"J. F.","familyName":"Ormes","affiliation":[]},{"name":"Paneque, D.","nameType":"Personal","givenName":"D.","familyName":"Paneque","affiliation":[]},{"name":"Panetta, J. H.","nameType":"Personal","givenName":"J. H.","familyName":"Panetta","affiliation":[]},{"name":"Parent, D.","nameType":"Personal","givenName":"D.","familyName":"Parent","affiliation":[]},{"name":"Perkins, J. S.","nameType":"Personal","givenName":"J. S.","familyName":"Perkins","affiliation":[]},{"name":"Pesce-Rollins, M.","nameType":"Personal","givenName":"M.","familyName":"Pesce-Rollins","affiliation":[]},{"name":"Pierbattista, M.","nameType":"Personal","givenName":"M.","familyName":"Pierbattista","affiliation":[]},{"name":"Piron, F.","nameType":"Personal","givenName":"F.","familyName":"Piron","affiliation":[]},{"name":"Pivato, G.","nameType":"Personal","givenName":"G.","familyName":"Pivato","affiliation":[]},{"name":"Pletsch, H. J.","nameType":"Personal","givenName":"H. J.","familyName":"Pletsch","affiliation":[]},{"name":"Porter, T. A.","nameType":"Personal","givenName":"T. A.","familyName":"Porter","affiliation":[]},{"name":"Possenti, A.","nameType":"Personal","givenName":"A.","familyName":"Possenti","affiliation":[]},{"name":"Rainò, S.","nameType":"Personal","givenName":"S.","familyName":"Rainò","affiliation":[]},{"name":"Rando, R.","nameType":"Personal","givenName":"R.","familyName":"Rando","affiliation":[]},{"name":"Ransom, S. M.","nameType":"Personal","givenName":"S. M.","familyName":"Ransom","affiliation":[]},{"name":"Ray, P. S.","nameType":"Personal","givenName":"P. S.","familyName":"Ray","affiliation":[]},{"name":"Razzano, M.","nameType":"Personal","givenName":"M.","familyName":"Razzano","affiliation":[]},{"name":"Rea, N.","nameType":"Personal","givenName":"N.","familyName":"Rea","affiliation":[]},{"name":"Reimer, A.","nameType":"Personal","givenName":"A.","familyName":"Reimer","affiliation":[]},{"name":"Reimer, O.","nameType":"Personal","givenName":"O.","familyName":"Reimer","affiliation":[]},{"name":"Renault, N.","nameType":"Personal","givenName":"N.","familyName":"Renault","affiliation":[]},{"name":"Reposeur, T.","nameType":"Personal","givenName":"T.","familyName":"Reposeur","affiliation":[]},{"name":"Ritz, S.","nameType":"Personal","givenName":"S.","familyName":"Ritz","affiliation":[]},{"name":"Romani, R. W.","nameType":"Personal","givenName":"R. W.","familyName":"Romani","affiliation":[]},{"name":"Roth, M.","nameType":"Personal","givenName":"M.","familyName":"Roth","affiliation":[]},{"name":"Rousseau, R.","nameType":"Personal","givenName":"R.","familyName":"Rousseau","affiliation":[]},{"name":"Roy, J.","nameType":"Personal","givenName":"J.","familyName":"Roy","affiliation":[]},{"name":"Ruan, J.","nameType":"Personal","givenName":"J.","familyName":"Ruan","affiliation":[]},{"name":"Sartori, A.","nameType":"Personal","givenName":"A.","familyName":"Sartori","affiliation":[]},{"name":"Saz Parkinson, P. M.","nameType":"Personal","givenName":"P. M.","familyName":"Saz Parkinson","affiliation":[]},{"name":"Scargle, J. D.","nameType":"Personal","givenName":"J. D.","familyName":"Scargle","affiliation":[]},{"name":"Schulz, A.","nameType":"Personal","givenName":"A.","familyName":"Schulz","affiliation":["Deutsches Elektronen-Synchrotron"]},{"name":"Sgrò, C.","nameType":"Personal","givenName":"C.","familyName":"Sgrò","affiliation":[]},{"name":"Shannon, R.","nameType":"Personal","givenName":"R.","familyName":"Shannon","affiliation":[]},{"name":"Siskind, E. J.","nameType":"Personal","givenName":"E. J.","familyName":"Siskind","affiliation":[]},{"name":"Smith, D. A.","nameType":"Personal","givenName":"D. A.","familyName":"Smith","affiliation":[]},{"name":"Spandre, G.","nameType":"Personal","givenName":"G.","familyName":"Spandre","affiliation":[]},{"name":"Spinelli, P.","nameType":"Personal","givenName":"P.","familyName":"Spinelli","affiliation":[]},{"name":"Stappers, B. W.","nameType":"Personal","givenName":"B. W.","familyName":"Stappers","affiliation":[]},{"name":"Strong, A. W.","nameType":"Personal","givenName":"A. W.","familyName":"Strong","affiliation":[]},{"name":"Suson, D. J.","nameType":"Personal","givenName":"D. J.","familyName":"Suson","affiliation":[]},{"name":"Takahashi, H.","nameType":"Personal","givenName":"H.","familyName":"Takahashi","affiliation":[]},{"name":"Thayer, J. G.","nameType":"Personal","givenName":"J. G.","familyName":"Thayer","affiliation":[]},{"name":"Thayer, J. B.","nameType":"Personal","givenName":"J. B.","familyName":"Thayer","affiliation":[]},{"name":"Theureau, G.","nameType":"Personal","givenName":"G.","familyName":"Theureau","affiliation":[]},{"name":"Thompson, D. J.","nameType":"Personal","givenName":"D. J.","familyName":"Thompson","affiliation":[]},{"name":"Thorsett, S. E.","nameType":"Personal","givenName":"S. E.","familyName":"Thorsett","affiliation":[]},{"name":"Tibaldo, L.","nameType":"Personal","givenName":"L.","familyName":"Tibaldo","affiliation":[]},{"name":"Tibolla, O.","nameType":"Personal","givenName":"O.","familyName":"Tibolla","affiliation":[]},{"name":"Tinivella, M.","nameType":"Personal","givenName":"M.","familyName":"Tinivella","affiliation":[]},{"name":"Torres, D. F.","nameType":"Personal","givenName":"D. F.","familyName":"Torres","affiliation":[]},{"name":"Tosti, G.","nameType":"Personal","givenName":"G.","familyName":"Tosti","affiliation":[]},{"name":"Troja, E.","nameType":"Personal","givenName":"E.","familyName":"Troja","affiliation":[]},{"name":"Uchiyama, Y.","nameType":"Personal","givenName":"Y.","familyName":"Uchiyama","affiliation":[]},{"name":"Usher, T. L.","nameType":"Personal","givenName":"T. L.","familyName":"Usher","affiliation":[]},{"name":"Vandenbroucke, J.","nameType":"Personal","givenName":"J.","familyName":"Vandenbroucke","affiliation":[]},{"name":"Vasileiou, V.","nameType":"Personal","givenName":"V.","familyName":"Vasileiou","affiliation":[]},{"name":"Venter, C.","nameType":"Personal","givenName":"C.","familyName":"Venter","affiliation":[]},{"name":"Vianello, G.","nameType":"Personal","givenName":"G.","familyName":"Vianello","affiliation":[]},{"name":"Vitale, V.","nameType":"Personal","givenName":"V.","familyName":"Vitale","affiliation":[]},{"name":"Wang, N.","nameType":"Personal","givenName":"N.","familyName":"Wang","affiliation":[]},{"name":"Weltevrede, P.","nameType":"Personal","givenName":"P.","familyName":"Weltevrede","affiliation":[]},{"name":"Winer, B. L.","nameType":"Personal","givenName":"B. L.","familyName":"Winer","affiliation":[]},{"name":"Wolff, M. T.","nameType":"Personal","givenName":"M. T.","familyName":"Wolff","affiliation":[]},{"name":"Wood, D. L.","nameType":"Personal","givenName":"D. L.","familyName":"Wood","affiliation":[]},{"name":"Wood, K. S.","nameType":"Personal","givenName":"K. S.","familyName":"Wood","affiliation":[]},{"name":"Wood, M.","nameType":"Personal","givenName":"M.","familyName":"Wood","affiliation":[]},{"name":"Yang, Z.","nameType":"Personal","givenName":"Z.","familyName":"Yang","affiliation":[]}],"titles":[{"title":"THE SECOND FERMI LARGE AREA TELESCOPE CATALOG OF GAMMA-RAY PULSARS"}],"publisher":"Deutsches Elektronen-Synchrotron, DESY, Hamburg","container":{"type":"Series","title":"The astrophysical journal / Supplement series 208(2)","lastPage":"59 (2013). doi:10.1088/0067","firstPage":"1","identifier":"0067-0049","identifierType":"ISSN"},"publicationYear":2013,"subjects":[{"subject":"520","schemeUri":"http://dewey.info/","subjectScheme":"dewey"}],"contributors":[],"dates":[{"date":"2013","dateType":"Copyrighted"},{"date":"2013","dateType":"Issued"}],"language":"en","types":{"ris":"JOUR","bibtex":"article","citeproc":"article-journal","schemaOrg":"ScholarlyArticle","resourceType":"Journal article","resourceTypeGeneral":"Text"},"relatedIdentifiers":[{"relationType":"IsVariantFormOf","relatedIdentifier":"10.1088/0067-0049/208/2/17","relatedIdentifierType":"DOI"},{"relationType":"IsPartOf","relatedIdentifier":"0067-0049","relatedIdentifierType":"ISSN"},{"relationType":"IsPartOf","relatedIdentifier":"1538-4365","relatedIdentifierType":"ISSN"}],"sizes":["pages 1-59"],"formats":[],"version":null,"rightsList":[],"descriptions":[{"description":"The astrophysical journal / Supplement series 208(2), 1-59 (2013). doi:10.1088/0067-0049/208/2/17","descriptionType":"SeriesInformation"},{"description":"Published by Institute of Physics Publ., London","descriptionType":"Other"}],"geoLocations":[],"fundingReferences":[],"url":"http://bib-pubdb1.desy.de/record/165916","contentUrl":null,"metadataVersion":0,"schemaVersion":"http://datacite.org/schema/kernel-3","source":null,"isActive":true,"state":"findable","reason":null,"created":"2018-06-28T09:19:08.000Z","registered":"2018-06-28T09:19:09.000Z","published":"2013","updated":"2019-02-03T02:31:46.000Z"},"relationships":{"client":{"data":{"id":"tib.desy","type":"clients"}}}}],"included":[{"id":"tib.desy","type":"clients","attributes":{"name":"DESY - Deutsches Elektronen-Synchrotron","symbol":"TIB.DESY","year":2011,"contactEmail":"Kirsten.Sachs@desy.de","alternateName":null,"description":null,"language":[],"clientType":"repository","domains":"desy.de,cern.ch","re3data":null,"opendoar":null,"issn":null,"url":null,"created":"2011-08-12T09:39:17.000Z","updated":"2018-08-26T02:35:08.000Z","isActive":true,"hasPassword":true},"relationships":{"provider":{"data":{"id":"tib","type":"providers"}},"prefixes":{"data":[{"id":"10.3204","type":"prefixes"}]}}}],"meta":{"total":1,"totalPages":1,"states":[{"id":"findable","title":"Findable","count":1}],"resourceTypes":[{"id":"text","title":"Text","count":1}],"created":[{"id":"2018","title":"2018","count":1}],"registered":[{"id":"2018","title":"2018","count":1}],"providers":[{"id":"tib","title":"German National Library of Science and Technology","count":1}],"clients":[{"id":"tib.desy","title":"DESY - Deutsches Elektronen-Synchrotron","count":1}],"affiliations":[],"prefixes":[{"id":"10.3204","title":"10.3204","count":1}],"certificates":[],"schemaVersions":[{"id":"3","title":"Schema 3","count":1}],"sources":[],"linkChecksStatus":[],"linksChecked":0,"linksWithSchemaOrg":[{"id":"0","title":"0","count":1}],"linkChecksSchemaOrgId":0,"linkChecksDcIdentifier":0,"linkChecksCitationDoi":0,"subjects":[{"id":"520","title":"520","count":1}]},"links":{"self":"https://api.datacite.org/dois?page%5Bcursor%5D=1&page%5Bsize%5D=50&query=updated%3A%5B2019-02-03T00%3A00%3A00.000Z+TO+2019-02-03T23%3A59%3A59.999Z%5D"}}
\ No newline at end of file diff --git a/python/tests/files/pubmed_31393839.xml b/python/tests/files/pubmed_31393839.xml new file mode 100644 index 00000000..644a02ec --- /dev/null +++ b/python/tests/files/pubmed_31393839.xml @@ -0,0 +1,95 @@ +<PubmedArticle> + <MedlineCitation Status="MEDLINE" IndexingMethod="Automated" Owner="NLM"> + <PMID Version="1">31393839</PMID> + <DateCompleted> + <Year>2019</Year> + <Month>12</Month> + <Day>17</Day> + </DateCompleted> + <DateRevised> + <Year>2019</Year> + <Month>12</Month> + <Day>17</Day> + </DateRevised> + <Article PubModel="Print"> + <Journal> + <ISSN IssnType="Print">0022-3875</ISSN> + <JournalIssue CitedMedium="Print"> + <Volume>53</Volume> + <Issue>6</Issue> + <PubDate> + <MedlineDate>2019</MedlineDate> + </PubDate> + </JournalIssue> + <Title>Journal of clinical orthodontics : JCO</Title> + <ISOAbbreviation>J Clin Orthod</ISOAbbreviation> + </Journal> + <ArticleTitle>Career-Turning Moments.</ArticleTitle> + <Pagination> + <MedlinePgn>325</MedlinePgn> + </Pagination> + <AuthorList CompleteYN="Y"> + <Author ValidYN="Y"> + <LastName>Keim</LastName> + <ForeName>Robert G</ForeName> + <Initials>RG</Initials> + <AffiliationInfo> + <Affiliation>J Clin Orthod, Greenwood Village, CO.</Affiliation> + </AffiliationInfo> + </Author> + </AuthorList> + <Language>eng</Language> + <PublicationTypeList> + <PublicationType UI="D016428">Journal Article</PublicationType> + </PublicationTypeList> + </Article> + <MedlineJournalInfo> + <Country>United States</Country> + <MedlineTA>J Clin Orthod</MedlineTA> + <NlmUniqueID>0243471</NlmUniqueID> + <ISSNLinking>0022-3875</ISSNLinking> + </MedlineJournalInfo> + <CitationSubset>D</CitationSubset> + <MeshHeadingList> + <MeshHeading> + <DescriptorName UI="D001696" MajorTopicYN="N">Biomechanical Phenomena</DescriptorName> + </MeshHeading> + <MeshHeading> + <DescriptorName UI="D002321" MajorTopicYN="Y">Career Choice</DescriptorName> + </MeshHeading> + <MeshHeading> + <DescriptorName UI="D006801" MajorTopicYN="N">Humans</DescriptorName> + </MeshHeading> + </MeshHeadingList> + </MedlineCitation> + <PubmedData> + <History> + <PubMedPubDate PubStatus="entrez"> + <Year>2019</Year> + <Month>8</Month> + <Day>9</Day> + <Hour>6</Hour> + <Minute>0</Minute> + </PubMedPubDate> + <PubMedPubDate PubStatus="pubmed"> + <Year>2019</Year> + <Month>8</Month> + <Day>9</Day> + <Hour>6</Hour> + <Minute>0</Minute> + </PubMedPubDate> + <PubMedPubDate PubStatus="medline"> + <Year>2019</Year> + <Month>12</Month> + <Day>18</Day> + <Hour>6</Hour> + <Minute>0</Minute> + </PubMedPubDate> + </History> + <PublicationStatus>ppublish</PublicationStatus> + <ArticleIdList> + <ArticleId IdType="pubmed">31393839</ArticleId> + </ArticleIdList> + </PubmedData> +</PubmedArticle> + diff --git a/python/tests/harvest_datacite.py b/python/tests/harvest_datacite.py new file mode 100644 index 00000000..926d67ba --- /dev/null +++ b/python/tests/harvest_datacite.py @@ -0,0 +1,45 @@ + +import json +import pytest +import datetime +import responses +from fatcat_tools.harvest import * + + +@responses.activate +def test_datacite_harvest_date(mocker): + + # mock out the harvest state object so it doesn't try to actually connect + # to Kafka + mocker.patch('fatcat_tools.harvest.harvest_common.HarvestState.initialize_from_kafka') + + # mock day request to crossref API + with open('tests/files/datacite_api.json', 'r') as f: + crossref_resp = json.loads(f.readline()) + responses.add(responses.GET, 'https://api.datacite.org/dois', + json=crossref_resp, status=200) + + harvester = HarvestDataciteWorker( + kafka_hosts="dummy", + produce_topic="dummy-produce-topic", + state_topic="dummy-state-topic", + contact_email="test@fatcat.wiki", + ) + + harvester.producer = mocker.Mock() + + harvester.fetch_date(datetime.date(2019, 2, 3)) + + assert len(responses.calls) == 1 + + # ensure email was included in User-Agent + assert "mailto:test@fatcat.wiki" in responses.calls[0].request.headers['User-Agent'] + + # check that correct date param was passed as expected + assert "query=updated%3A%5B2019-02-03T00%3A00%3A00.000Z+TO+2019-02-03T23%3A59%3A59.999Z%5D" in responses.calls[0].request.url + + # check that we published the expected number of DOI objects were published + # to the (mock) kafka topic + assert harvester.producer.produce.call_count == 1 + assert harvester.producer.flush.call_count == 1 + assert harvester.producer.poll.called_once_with(0) diff --git a/python/tests/import_pubmed.py b/python/tests/import_pubmed.py index 0d551d02..49609f75 100644 --- a/python/tests/import_pubmed.py +++ b/python/tests/import_pubmed.py @@ -119,3 +119,10 @@ def test_pubmed_xml_parse(pubmed_importer): assert r2.refs[0].extra['unstructured'] == "Microbiology. 2009 Jun;155(Pt 6):1840-6" assert r2.refs[0].extra['pmid'] == "19383690" +def test_pubmed_xml_dates(pubmed_importer): + with open('tests/files/pubmed_31393839.xml', 'r') as f: + soup = BeautifulSoup(f, "xml") + r1 = pubmed_importer.parse_record(soup.find_all("PubmedArticle")[0]) + + assert r1.release_year == 2019 + |