diff options
75 files changed, 3512 insertions, 930 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d4428d..d2c3be52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -290,7 +290,7 @@ changes. in editgroup objects by default. - `created` timestamp is included in editgroup objects (when GET) by default. - in client libraries, `FileEntityUrls` renamed `FileUrl`, and several similar - singlular/plural and `Entity` removed renamings. + singular/plural and `Entity` removed renamings. - elasticsearch release schema updated to match API schema changes ### Removed diff --git a/extra/bulk_edits/2022-03-08_chocula.md b/extra/bulk_edits/2022-03-08_chocula.md new file mode 100644 index 00000000..1877a236 --- /dev/null +++ b/extra/bulk_edits/2022-03-08_chocula.md @@ -0,0 +1,31 @@ + +Periodic import of chocula metadata updates. + +## Prod Import + + date + # Wed Mar 9 02:13:55 UTC 2022 + + git log -n1 + # commit 72e3825893ae614fcd6c6ae8a513745bfefe36b2 + + export FATCAT_AUTH_WORKER_JOURNAL_METADATA=[...] + head -n100 /srv/fatcat/datasets/chocula_fatcat_export.2022-03-08.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 100, 'exists': 85, 'exists-skip-update': 85, 'update': 14, 'insert': 1, 'skip': 0}) + +Some of these are just "as of" date updates on DOAJ metadata, but most are +"good". Lots of KBART holding dates incremented by a year (to include 2022). + + time cat /srv/fatcat/datasets/chocula_fatcat_export.2022-03-08.json | ./fatcat_import.py chocula --do-updates - + + + Counter({'total': 184950, 'exists': 151925, 'exists-skip-update': 151655, 'update': 29953, 'insert': 3072 + , 'exists-by-issnl': 270, 'skip': 0}) + + real 11m7.011s + user 4m48.705s + sys 0m16.761s + +Great! + +Now update stats, following `extra/container_count_update/README.md`. diff --git a/extra/bulk_edits/2022-03-08_doaj.md b/extra/bulk_edits/2022-03-08_doaj.md new file mode 100644 index 00000000..fc6438d5 --- /dev/null +++ b/extra/bulk_edits/2022-03-08_doaj.md @@ -0,0 +1,23 @@ + +Simple periodic update of DOAJ article-level metadata. + + cat doaj_article_data_*/article_batch*.json | jq .[] -c | pv -l | gzip > doaj_article_data_2021-05-25_all.json.gz + => 6.1M 0:18:45 [5.42k/s] + => 7.26M 0:30:45 [3.94k/s] + + export FATCAT_AUTH_WORKER_DOAJ=... + cat /srv/fatcat/tasks/doaj_article_data_2022-03-07_sample_10k.json | ./fatcat_import.py doaj-article --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt - + # Counter({'total': 10000, 'exists': 8827, 'exists-fuzzy': 944, 'insert': 219, 'skip': 8, 'skip-title': 8, 'skip-doaj-id-mismatch': 2, 'update': 0}) + + zcat /srv/fatcat/tasks/doaj_article_data_2022-03-07_all.json.gz | shuf | pv -l | parallel -j12 --round-robin --pipe ./fatcat_import.py doaj-article --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt - + +The above seemed to use too much CPU, and caused a brief outage. Very high CPU +use for just the python import processes, for whatever reason. Turned down +parallelism and trying again: + + zcat /srv/fatcat/tasks/doaj_article_data_2022-03-07_all.json.gz | pv -l | parallel -j6 --round-robin --pipe ./fatcat_import.py doaj-article --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt - + # multiple counts of: + # Counter({'total': 1196313, 'exists': 1055412, 'exists-fuzzy': 111490, 'insert': 27835, 'skip': 1280, 'skip-title': 1280, 'skip-doaj-id-mismatch': 296, 'update': 0}) + # estimated only 167,010 new entities + +Then did a follow-up sandcrawler ingest, see notes in that repository. diff --git a/extra/bulk_edits/2022-04-07_initial_datasets.md b/extra/bulk_edits/2022-04-07_initial_datasets.md new file mode 100644 index 00000000..90827a38 --- /dev/null +++ b/extra/bulk_edits/2022-04-07_initial_datasets.md @@ -0,0 +1,22 @@ + +Importing fileset and file entities from initial sandcrawler ingests. + +Git commit: `ede98644a89afd15d903061e0998dbd08851df6d` + +Filesets: + + export FATCAT_AUTH_SANDCRAWLER=[...] + cat /tmp/ingest_dataset_combined_results.2022-04-04.partial.json \ + | ./fatcat_import.py ingest-fileset-results - + # editgroup_5l47i7bscvfmpf4ddytauoekea + # Counter({'total': 195, 'skip': 176, 'skip-hit': 160, 'insert': 19, 'skip-single-file': 14, 'skip-partial-file-info': 2, 'update': 0, 'exists': 0}) + + cat /srv/fatcat/datasets/ingest_dataset_combined_results.2022-04-04.partial.json \ + | ./fatcat_import.py ingest-fileset-file-results - + # editgroup_i2k2ucon7nap3gui3z7amuiug4 + # Counter({'total': 195, 'skip': 184, 'skip-hit': 160, 'skip-status': 24, 'insert': 11, 'update': 0, 'exists': 0}) + +Tried running again, to ensure that there are not duplicate inserts, and that +worked ('exists' instead of 'insert' counts). + +Finally! diff --git a/extra/bulk_edits/2022-04-20_isiarticles.md b/extra/bulk_edits/2022-04-20_isiarticles.md new file mode 100644 index 00000000..b0177a46 --- /dev/null +++ b/extra/bulk_edits/2022-04-20_isiarticles.md @@ -0,0 +1,39 @@ + +See metadata cleanups for context. Basically a couple tens of thousands of sample/spam articles hosted on the domain isiarticles.com. + +## Prod Updates + +Start small: + + export FATCAT_API_HOST=https://api.fatcat.wiki + export FATCAT_AUTH_WORKER_CLEANUP=[...] + export FATCAT_API_AUTH_TOKEN=$FATCAT_AUTH_WORKER_CLEANUP + + fatcat-cli search file domain:isiarticles.com --entity-json -n0 \ + | rg -v '"content_scope"' \ + | rg 'isiarticles.com/' \ + | head -n50 \ + | pv -l \ + | fatcat-cli batch update file release_ids= content_scope=sample --description 'Un-link and mark isiarticles PDFs as content_scope=sample' --auto-accept + # editgroup_ihx75kzsebgzfisgjrv67zew5e + +The full batch: + + fatcat-cli search file domain:isiarticles.com --entity-json -n0 \ + | rg -v '"content_scope"' \ + | rg 'isiarticles.com/' \ + | pv -l \ + | fatcat-cli batch update file release_ids= content_scope=sample --description 'Un-link and mark isiarticles PDFs as content_scope=sample' --auto-accept + +And some more with ':80' in the URL: + + fatcat-cli search file domain:isiarticles.com '!content_scope:*' --entity-json -n0 \ + | rg -v '"content_scope"' \ + | rg 'isiarticles.com:80/' \ + | pv -l \ + | fatcat-cli batch update file release_ids= content_scope=sample --description 'Un-link and mark isiarticles PDFs as content_scope=sample' --auto-accept + +Verify: + + fatcat-cli search file domain:isiarticles.com '!content_scope:*' --count + 0 diff --git a/extra/bulk_edits/2022-07-06_chocula.md b/extra/bulk_edits/2022-07-06_chocula.md new file mode 100644 index 00000000..86bf36fb --- /dev/null +++ b/extra/bulk_edits/2022-07-06_chocula.md @@ -0,0 +1,25 @@ + +Periodic import of chocula metadata updates. + +## Prod Import + + date + # Wed Jul 6 23:29:47 UTC 2022 + + git log -n1 + # aff3f40a5177dd6de4eee8ea7bca78df7a595bf3 + + export FATCAT_AUTH_WORKER_JOURNAL_METADATA=[...] + head -n100 /srv/fatcat/datasets/chocula_fatcat_export.2022-07-06.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 100, 'exists': 86, 'exists-skip-update': 83, 'update': 13, 'exists-by-issnl': 3, 'insert': 1, 'skip': 0}) + +Many updates are just KBART holding dates or DOAJ as-of dates, but that is fine +and expected. + + time cat /srv/fatcat/datasets/chocula_fatcat_export.2022-07-06.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 187480, 'exists': 155943, 'exists-skip-update': 151171, 'update': 30437, 'exists-by-issnl': 4772, 'insert': 1100, 'skip': 0}) + # real 10m28.081s + # user 4m37.447s + # sys 0m16.063s + +Now update stats, following `extra/container_count_update/README.md`. diff --git a/extra/bulk_edits/2022-07-12_cleanup_doaj_missing_container_id.md b/extra/bulk_edits/2022-07-12_cleanup_doaj_missing_container_id.md new file mode 100644 index 00000000..b17e799d --- /dev/null +++ b/extra/bulk_edits/2022-07-12_cleanup_doaj_missing_container_id.md @@ -0,0 +1,38 @@ + +There is a batch of about 480 releases with DOAJ identifiers but no container +linkage. These seem to all be from the same actual container: + + fatcat-cli search releases 'doaj_id:*' '!container_id:*' --count + # 486 + + fatcat-cli search releases 'doaj_id:*' '!container_id:*' --index-json -n 0 | jq .containe + # Got 486 hits in 138ms + # "Revista de Sistemas, Cibernética e Informática" + +Edit pipeline: + + export FATCAT_AUTH_WORKER_CLEANUP=[...] + export FATCAT_API_AUTH_TOKEN=$FATCAT_AUTH_WORKER_CLEANUP + + # start small + fatcat-cli search releases 'doaj_id:*' '!container_id:*' 'journal:Cibernética' --entity-json --limit 50 \ + | jq 'select(.container_id == null)' -c \ + | rg 'Cibernética' \ + | fatcat-cli batch update release container_id=ubwuhr4obzgr7aadszhurhef5m --description "Add container linkage for DOAJ articles with ISSN 1690-8627" + # editgroup_g2zrm3wkmneoldtqfxpbkaoeh4 + +Looks good, merged. + + # full auto + fatcat-cli search releases 'doaj_id:*' '!container_id:*' 'journal:Cibernética' --entity-json --limit 500 \ + | jq 'select(.container_id == null)' -c \ + | rg 'Cibernética' \ + | fatcat-cli batch update release container_id=ubwuhr4obzgr7aadszhurhef5m --description "Add container linkage for DOAJ articles with ISSN 1690-8627" --auto-accept + +Verify: + + fatcat-cli search releases 'doaj_id:*' '!container_id:*' --count + # 0 + +Also planning to have DOAJ article importer 'skip' in the future for articles +with no `container_id` match. diff --git a/extra/bulk_edits/2022-07-12_jalc.md b/extra/bulk_edits/2022-07-12_jalc.md new file mode 100644 index 00000000..d9f09fee --- /dev/null +++ b/extra/bulk_edits/2022-07-12_jalc.md @@ -0,0 +1,47 @@ + +Import of a 2022-04 JALC DOI metadata snapshot. + +Note that we had downloaded a prior 2021-04 snapshot, but don't seem to have +ever imported it. + +## Download and Archive + +URL for bulk snapshot is available at the bottom of this page: <https://form.jst.go.jp/enquetes/jalcmetadatadl_1703> + +More info: <http://japanlinkcenter.org/top/service/service_data.html> + + wget 'https://japanlinkcenter.org/lod/JALC-LOD-20220401.gz?jalcmetadatadl_1703' + wget 'http://japanlinkcenter.org/top/doc/JaLC_LOD_format.pdf' + wget 'http://japanlinkcenter.org/top/doc/JaLC_LOD_sample.pdf' + + mv 'JALC-LOD-20220401.gz?jalcmetadatadl_1703' JALC-LOD-20220401.gz + + ia upload jalc-bulk-metadata-2022-04 -m collection:ia_biblio_metadata jalc_logo.png JALC-LOD-20220401.gz JaLC_LOD_format.pdf JaLC_LOD_sample.pdf + +## Import + +As of 2022-07-19, 6,502,202 release hits for `doi_registrar:jalc`. + +Re-download the file: + + cd /srv/fatcat/datasets + wget 'https://archive.org/download/jalc-bulk-metadata-2022-04/JALC-LOD-20220401.gz' + gunzip JALC-LOD-20220401.gz + cd /srv/fatcat/src/python + + wc -l /srv/fatcat/datasets/JALC-LOD-20220401 + 9525225 + +Start with some samples: + + export FATCAT_AUTH_WORKER_JALC=[...] + shuf -n100 /srv/fatcat/datasets/JALC-LOD-20220401 | ./fatcat_import.py --batch-size 100 jalc - /srv/fatcat/datasets/ISSN-to-ISSN-L.txt + # Counter({'total': 100, 'exists': 89, 'insert': 11, 'skip': 0, 'update': 0}) + +Full import (single threaded): + + cat /srv/fatcat/datasets/JALC-LOD-20220401 | pv -l | ./fatcat_import.py --batch-size 100 jalc - /srv/fatcat/datasets/ISSN-to-ISSN-L.txt + # 9.53M 22:26:06 [ 117 /s] + # Counter({'total': 9510096, 'exists': 8589731, 'insert': 915032, 'skip': 5333, 'inserted.container': 119, 'update': 0}) + +Wow, almost a million new releases! 7,417,245 results for `doi_registrar:jalc`. diff --git a/extra/bulk_edits/2022-07-12_orcid.md b/extra/bulk_edits/2022-07-12_orcid.md new file mode 100644 index 00000000..760a16c8 --- /dev/null +++ b/extra/bulk_edits/2022-07-12_orcid.md @@ -0,0 +1,64 @@ + +Annual ORCID import, using 2021 public data file. Didn't do this last year, so +a catch-up, and will need to do another update later in 2022 (presumably in +November/December). + +Not sure how many records this year. Current count on the orcid.org website is +over 14 million ORCIDs, in July 2022. + +Files download from: + +- <https://info.orcid.org/orcids-2021-public-data-file-is-now-available> +- <https://orcid.figshare.com/articles/dataset/ORCID_Public_Data_File_2021/16750535> +- <https://archive.org/details/orcid-dump-2021> + +## Prep + + ia upload orcid-dump-2021 -m collection:ia_biblio_metadata ORCID_2021_10_* orcid-logo.png + + wget https://github.com/ORCID/orcid-conversion-lib/raw/master/target/orcid-conversion-lib-3.0.7-full.jar + + java -jar orcid-conversion-lib-3.0.7-full.jar --tarball -i ORCID_2021_10_summaries.tar.gz -v v3_0 -o ORCID_2021_10_summaries_json.tar.gz + + tar xvf ORCID_2021_10_summaries_json.tar.gz + + fd .json ORCID_2021_10_summaries/ | parallel cat {} | jq . -c | pv -l | gzip > ORCID_2021_10_summaries.json.gz + # 12.6M 27:59:25 [ 125 /s] + + zcat ORCID_2021_10_summaries.json.gz | shuf -n10000 | gzip > ORCID_2021_10_summaries.sample_10k.json.gz + + ia upload orcid-dump-2021 ORCID_2021_10_summaries.json.gz ORCID_2021_10_summaries.sample_10k.json.gz + +## Import + +Fetch to prod machine: + + wget https://archive.org/download/orcid-dump-2021/ORCID_2021_10_summaries.json.gz + wget https://archive.org/download/orcid-dump-2021/ORCID_2021_10_summaries.sample_10k.json.gz + +Sample: + + export FATCAT_AUTH_WORKER_ORCID=[...] + zcat /srv/fatcat/datasets/ORCID_2021_10_summaries.sample_10k.json.gz | ./fatcat_import.py orcid - + # in 2020: Counter({'total': 10000, 'exists': 7356, 'insert': 2465, 'skip': 179, 'update': 0}) + # this time: Counter({'total': 10000, 'exists': 7577, 'insert': 2191, 'skip': 232, 'update': 0}) + +Bulk import: + + export FATCAT_AUTH_WORKER_ORCID=[...] + time zcat /srv/fatcat/datasets/ORCID_2021_10_summaries.json.gz | pv -l | parallel -j8 --round-robin --pipe ./fatcat_import.py orcid - + 12.6M 1:24:04 [2.51k/s] + Counter({'total': 1574111, 'exists': 1185437, 'insert': 347039, 'skip': 41635, 'update': 0}) + Counter({'total': 1583157, 'exists': 1193341, 'insert': 348187, 'skip': 41629, 'update': 0}) + Counter({'total': 1584441, 'exists': 1193385, 'insert': 349424, 'skip': 41632, 'update': 0}) + Counter({'total': 1575971, 'exists': 1187270, 'insert': 347190, 'skip': 41511, 'update': 0}) + Counter({'total': 1577323, 'exists': 1188892, 'insert': 346759, 'skip': 41672, 'update': 0}) + Counter({'total': 1586719, 'exists': 1195610, 'insert': 349115, 'skip': 41994, 'update': 0}) + Counter({'total': 1578484, 'exists': 1189423, 'insert': 347276, 'skip': 41785, 'update': 0}) + Counter({'total': 1578728, 'exists': 1190316, 'insert': 346445, 'skip': 41967, 'update': 0}) + + real 84m5.297s + user 436m26.428s + sys 41m36.959s + +Roughly 2.7 million new ORCIDs, great! diff --git a/extra/bulk_edits/2022-07-13_dblp.md b/extra/bulk_edits/2022-07-13_dblp.md new file mode 100644 index 00000000..25405132 --- /dev/null +++ b/extra/bulk_edits/2022-07-13_dblp.md @@ -0,0 +1,114 @@ + +## Prep + + 2022-07-13 05:24:33 (177 KB/s) - ‘dblp.xml.gz’ saved [715701831/715701831] + + Counter({'total': 9186263, 'skip': 9186263, 'has-doi': 4960506, 'skip-key-type': 3037457, 'skip-arxiv-corr': 439104, 'skip-title': 1, 'insert': 0, 'update': 0, 'exists': 0}) + 5.71M 3:37:38 [ 437 /s] + + 7.48k 0:38:18 [3.25 /s] + + +## Container Import + +Run 2022-07-15, after a database backup/snapshot. + + export FATCAT_AUTH_WORKER_DBLP=[...] + ./fatcat_import.py dblp-container --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt --dblp-container-map-file ../extra/dblp/existing_dblp_containers.tsv --dblp-container-map-output ../extra/dblp/all_dblp_containers.tsv ../extra/dblp/dblp_container_meta.json + # Got 5310 existing dblp container mappings. + # Counter({'total': 7471, 'exists': 7130, 'insert': 341, 'skip': 0, 'update': 0}) + + wc -l existing_dblp_containers.tsv all_dblp_containers.tsv dblp_container_meta.json prefix_list.txt + 5310 existing_dblp_containers.tsv + 12782 all_dblp_containers.tsv + 7471 dblp_container_meta.json + 7476 prefix_list.txt + + +## Release Import + + export FATCAT_AUTH_WORKER_DBLP=[...] + ./fatcat_import.py dblp-release --dblp-container-map-file ../extra/dblp/all_dblp_containers.tsv ../extra/dblp/dblp.xml + # Got 7480 dblp container mappings. + + /1/srv/fatcat/src/python/fatcat_tools/importers/dblp_release.py:358: UserWarning: unexpected dblp ext_id match after lookup failed dblp=conf/gg/X90 ident=gfvkxubvsfdede7ps4af3oa34q + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/dblp_release.py:358: UserWarning: unexpected dblp ext_id match after lookup failed dblp=conf/visalg/X88 ident=lvfyrd3lvva3hjuaaokzyoscmm + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/dblp_release.py:358: UserWarning: unexpected dblp ext_id match after lookup failed dblp=conf/msr/PerumaANMO22 ident=2grlescl2bcpvd5yoc4npad3bm + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/dblp_release.py:358: UserWarning: unexpected dblp ext_id match after lookup failed dblp=conf/dagstuhl/Brodlie97 ident=l6nh222fpjdzfotchu7vfjh6qu + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/dblp_release.py:358: UserWarning: unexpected dblp ext_id match after lookup failed dblp=series/gidiss/2018 ident=x6t7ze4z55enrlq2dnac4qqbve + + Counter({'total': 9186263, 'exists': 5356574, 'has-doi': 4960506, 'skip': 3633039, 'skip-key-type': 3037457, 'skip-arxiv-corr': 439104, 'exists-fuzzy': 192376, 'skip-dblp-container-missing': 156477, 'insert': 4216, 'skip-arxiv': 53, 'skip-dblp-id-mismatch': 5, 'skip-title': 1, 'update': 0}) + +NOTE: had to re-try in the middle, so these counts not accurate overall. + +Seems like a large number of `skip-dblp-container-missing`. Maybe should have +re-generated that file differently? + +After this import there are 2,217,670 releases with a dblp ID, and 478,983 with +a dblp ID and no DOI. + + +## Sandcrawler Seedlist Generation + +Almost none of the ~487k dblp releases with no DOI have an associated file. +This implies that no ingest has happened yet, even though the fatcat importer +does parse and filter the "fulltext" URLs out of dblp records. + + cat dblp_releases_partial.json | pipenv run ./dblp2ingestrequest.py - | pv -l | gzip > dblp_sandcrawler_ingest_requests.json.gz + # 631k 0:02:39 [3.96k/s] + + zcat dblp_sandcrawler_ingest_requests.json.gz | jq -r .base_url | cut -f3 -d/ | sort | uniq -c | sort -nr | head -n25 + 43851 ceur-ws.org + 33638 aclanthology.org + 32077 aisel.aisnet.org + 31017 ieeexplore.ieee.org + 26426 dl.acm.org + 23817 hdl.handle.net + 22400 www.isca-speech.org + 20072 tel.archives-ouvertes.fr + 18609 www.aaai.org + 18244 eprint.iacr.org + 15720 ethos.bl.uk + 14727 nbn-resolving.org + 14470 proceedings.mlr.press + 14095 dl.gi.de + 12159 proceedings.neurips.cc + 10890 knowledge.amia.org + 10049 www.usenix.org + 9675 papers.nips.cc + 7541 subs.emis.de + 7396 openaccess.thecvf.com + 7345 mindmodeling.org + 6574 ojs.aaai.org + 5814 www.lrec-conf.org + 5773 search.ndltd.org + 5311 ijcai.org + +This is the first ingest, so let's do some sampling in the 'daily' queue: + + zcat dblp_sandcrawler_ingest_requests.json.gz + + zcat dblp_sandcrawler_ingest_requests.json.gz | shuf -n100 | rg -v "\\\\" | jq . -c | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-daily -p -1 + +Looks like we can probably get away with doing these in the daily ingest queue, +instead of bulk? Try a larger batch: + + zcat dblp_sandcrawler_ingest_requests.json.gz | shuf -n10000 | rg -v "\\\\" | jq . -c | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-daily -p -1 + +Nope, these are going to need bulk ingest then follow-up crawling. Will +heritrix crawl along with JALC and DOAJ stuff. + + zcat dblp_sandcrawler_ingest_requests.json.gz | rg -v "\\\\" | jq . -c | pv -l | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-bulk -p -1 + # 631k 0:00:11 [54.0k/s] + + +TODO: +x python or jq transform of JSON objects +x filter out german book/library URLs +x ensure fatcat importer will actually import dblp matches +x test with a small batch in daily or priority queue +- enqueue all in bulk mode, even if processed before? many probably MAG or OAI-PMH previously diff --git a/extra/bulk_edits/2022-07-19_doaj.md b/extra/bulk_edits/2022-07-19_doaj.md new file mode 100644 index 00000000..d25f2dda --- /dev/null +++ b/extra/bulk_edits/2022-07-19_doaj.md @@ -0,0 +1,78 @@ + +Doing a batch import of DOAJ articles. Will need to do another one of these +soon after setting up daily (OAI-PMH feed) ingest. + +## Prep + + wget https://doaj.org/csv + wget https://doaj.org/public-data-dump/journal + wget https://doaj.org/public-data-dump/article + + mv csv journalcsv__doaj_20220719_2135_utf8.csv + mv journal doaj_journal_data_2022-07-19.tar.gz + mv article doaj_article_data_2022-07-19.tar.gz + + ia upload doaj_data_2022-07-19 -m collection:ia_biblio_metadata ../logo_cropped.jpg journalcsv__doaj_20220719_2135_utf8.csv doaj_journal_data_2022-07-19.tar.gz doaj_article_data_2022-07-19.tar.gz + + tar xvf doaj_journal_data_2022-07-19.tar.gz + cat doaj_journal_data_*/journal_batch_*.json | jq .[] -c | pv -l | gzip > doaj_journal_data_2022-07-19_all.json.gz + + tar xvf doaj_article_data_2022-07-19.tar.gz + cat doaj_article_data_*/article_batch*.json | jq .[] -c | pv -l | gzip > doaj_article_data_2022-07-19_all.json.gz + + ia upload doaj_data_2022-07-19 doaj_journal_data_2022-07-19_all.json.gz doaj_article_data_2022-07-19_all.json.gz + +On fatcat machine: + + cd /srv/fatcat/datasets + wget https://archive.org/download/doaj_data_2022-07-19/doaj_article_data_2022-07-19_all.json.gz + +## Prod Article Import + + git rev: 582495f66e5e08b6e257360097807711e53008d4 + (includes DOAJ container-id required patch) + + date: Tue Jul 19 22:46:42 UTC 2022 + + `doaj_id:*`: 1,335,195 hits + +Start with sample: + + zcat /srv/fatcat/datasets/doaj_article_data_2022-07-19_all.json.gz | shuf -n1000 > /srv/fatcat/datasets/doaj_article_data_2022-07-19_sample.json + + export FATCAT_AUTH_WORKER_DOAJ=[...] + cat /srv/fatcat/datasets/doaj_article_data_2022-07-19_sample.json | pv -l | ./fatcat_import.py doaj-article --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt - + # Counter({'total': 1000, 'exists': 895, 'exists-fuzzy': 93, 'insert': 9, 'skip': 3, 'skip-no-container': 3, 'update': 0}) + +Pretty few imports. + +Full ingest: + + export FATCAT_AUTH_WORKER_DOAJ=[...] + zcat /srv/fatcat/datasets/doaj_article_data_2022-07-19_all.json.gz | pv -l | parallel -j6 --round-robin --pipe ./fatcat_import.py doaj-article --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt - + # Counter({'total': 1282908, 'exists': 1145439, 'exists-fuzzy': 117120, 'insert': 16357, 'skip': 3831, 'skip-no-container': 2641, 'skip-title': 1190, 'skip-doaj-id-mismatch': 161, 'update': 0}) + +Times 6x, around 100k releases added. + +Got a bunch of: + + /1/srv/fatcat/src/python/fatcat_tools/importers/doaj_article.py:233: UserWarning: unexpected DOAJ ext_id match after lookup failed doaj=fcdb7a7a9729403d8d99a21f6970dd1d ident=wesvmjwihvblzayfmrvvgr4ulm + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/doaj_article.py:233: UserWarning: unexpected DOAJ ext_id match after lookup failed doaj=1455dfe24583480883dbbb293a4bc0c6 ident=lfw57esesjbotms3grvvods5dq + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/doaj_article.py:233: UserWarning: unexpected DOAJ ext_id match after lookup failed doaj=88fa65a33c8e484091fc76f4cda59c25 ident=22abqt5qe5e7ngjd5fkyvzyc4q + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/doaj_article.py:233: UserWarning: unexpected DOAJ ext_id match after lookup failed doaj=eb7b03dc3dc340cea36891a68a50cce7 ident=ljedohlfyzdkxebgpcswjtd77q + warnings.warn(warn_str) + /1/srv/fatcat/src/python/fatcat_tools/importers/doaj_article.py:233: UserWarning: unexpected DOAJ ext_id match after lookup failed doaj=519617147ce248ea88d45ab098342153 ident=a63bqkttrbhyxavfr7li2w2xf4 + +Should investigate! + +Also, noticed that DOAJ importer is hitting `api.fatcat.wiki`, not the public +API endpoint. Guessing this is via fuzzycat. + +1,434,266 results for `doaj_id:*`. + +Then did a follow-up sandcrawler ingest, see notes in that repository. Note +that newer ingest can crawl doaj.org, bypassing the sandcrawler SQL load, but +the direct crawling is probably still faster. diff --git a/extra/bulk_edits/2022-07-29_chocula.md b/extra/bulk_edits/2022-07-29_chocula.md new file mode 100644 index 00000000..1f6f36ca --- /dev/null +++ b/extra/bulk_edits/2022-07-29_chocula.md @@ -0,0 +1,47 @@ + +Periodic import of chocula metadata updates. + +In particular, expecting a bunch of `publisher_type` updates. + +Going to explicitly not do DOAJ-only updates this time around. That is, if +container would have been updated, then new DOAJ 'extra' metadata will pass +through. But don't only update entity for this reason. This is to reduce churn +based only on the `as-of` key. Should probably change the behavior next time +around. + +## Prod Import + + date + # Sat Jul 30 01:18:41 UTC 2022 + + git log -n1 + # 5ecf72cbb488a9a50eb869ea55b4c2bfc1440731 + + diff --git a/python/fatcat_tools/importers/chocula.py b/python/fatcat_tools/importers/chocula.py + index 38802bcb..762c44dd 100644 + --- a/python/fatcat_tools/importers/chocula.py + +++ b/python/fatcat_tools/importers/chocula.py + @@ -139,7 +139,7 @@ class ChoculaImporter(EntityImporter): + if ce.extra.get("publisher_type") and not ce.extra.get("publisher_type"): + # many older containers were missing this metadata + do_update = True + - for k in ("kbart", "ia", "doaj"): + + for k in ("kbart", "ia"): + # always update these fields if not equal (chocula override) + if ce.extra.get(k) and ce.extra[k] != existing.extra.get(k): + do_update = True + + export FATCAT_AUTH_WORKER_JOURNAL_METADATA=[...] + shuf -n100 /srv/fatcat/datasets/chocula_fatcat_export.2022-07-30.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 100, 'exists': 98, 'exists-skip-update': 98, 'update': 2, 'skip': 0, 'insert': 0}) + + shuf -n1000 /srv/fatcat/datasets/chocula_fatcat_export.2022-07-30.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 1000, 'exists': 986, 'exists-skip-update': 986, 'update': 12, 'insert': 2, 'skip': 0}) + +Huh, not seeing any `publisher_type` changes, which I was expecting more of. + + time cat /srv/fatcat/datasets/chocula_fatcat_export.2022-07-30.json | ./fatcat_import.py chocula --do-updates - + # Counter({'total': 188506, 'exists': 185808, 'exists-skip-update': 185806, 'update': 2495, 'insert': 203, 'exists-by-issnl': 2, 'skip': 0}) + +Looking through the changelog, some did through with `publisher_type` updates. +Whew! diff --git a/extra/bulk_edits/CHANGELOG.md b/extra/bulk_edits/CHANGELOG.md index 278dc1d8..716c95d6 100644 --- a/extra/bulk_edits/CHANGELOG.md +++ b/extra/bulk_edits/CHANGELOG.md @@ -9,6 +9,48 @@ this file should probably get merged into the guide at some point. This file should not turn in to a TODO list! +## 2022-07 + +Ran a journal-level metadata update, using chocula. + +Cleaned up just under 500 releases with missing `container_id` from an older +DOAJ article import. + +Imported roughly 100k releases from DOAJ, new since 2022-04. + +Imported roughly 2.7 million new ORCiD `creator` entities, using the 2021 dump +(first update since 2020 dump). + +Imported almost 1 million new DOI release entities from JALC, first update in +more than a year. + +Imported at least 400 new dblp containers, and an unknown number of new dblp +release entities. + +Cleaned up about a thousand containers with incorrect `publisher_type`, based +on current publisher name. Further updates will populate after the next chocula +import. + +Ran a second batch of journal-level metadata updates, from chocula, resulting +in a couple thousand updated entities. + + +## 2022-04 + +Imported some initial fileset entities. + +Updated about 25k file entities from isiarticles.com, which are samples (spam +for translation service) to remove release linkage and set +`content_scope=sample` (similar to the springer "page one" case). + +## 2022-03 + +Ran a journal-level metadata update, using chocula. + +Run a DOAJ article-level metadata import, yielding a couple hundred thousand +new release entities. Crawling and bulk ingest of HTML and PDF fulltext for +these articles also started. + ## 2022-02 - removed `container_id` linkage for some Datacite DOI releases which are diff --git a/extra/cleanups/container_publisher_type.md b/extra/cleanups/container_publisher_type.md new file mode 100644 index 00000000..dba800d3 --- /dev/null +++ b/extra/cleanups/container_publisher_type.md @@ -0,0 +1,100 @@ + +A bunch of MDPI journals are incorrectly listed as 'longtail'. + + fatcat-cli search container 'publisher:mdpi publisher_type:* !publisher_type:oa' --count + # 245 + +Because this is 'extra' metadata, need a little python script to change the +metadata (fatcat-cli doesn't have this feature yet): + + import sys + import json + + publisher_type = sys.argv[1].strip().lower() + #print(publisher_type, file=sys.stderr) + + for line in sys.stdin: + if not line.strip(): + continue + container = json.loads(line) + container["extra"]["publisher_type"] = publisher_type + print(json.dumps(container)) + +Run some cleanups: + + export FATCAT_AUTH_WORKER_CLEANUP=[...] + export FATCAT_API_AUTH_TOKEN=$FATCAT_AUTH_WORKER_CLEANUP + + fatcat-cli search container 'publisher:mdpi publisher_type:* !publisher_type:oa' --entity-json --limit 50 \ + | jq 'select(.publisher_type != "oa")' -c \ + | python3 ./container_publisher_type.py oa \ + | fatcat-cli batch update container --description "Update container publisher_type" + # editgroup_oum6mnkl2rbn3jaua4a2gdlj5q + +Looks good, run the rest: + + fatcat-cli search container 'publisher:mdpi publisher_type:* !publisher_type:oa' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "oa")' -c \ + | python3 ./container_publisher_type.py oa \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + +Some more cleanup patterns: + + fatcat-cli search container 'publisher:"Frontiers Media SA" publisher_type:* !publisher_type:oa' --count + # 84 + + fatcat-cli search container 'publisher:"Frontiers Media SA" publisher_type:* !publisher_type:oa' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "oa")' -c \ + | python3 ./container_publisher_type.py oa \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:"Walter de Gruyter" publisher_type:* !publisher_type:commercial !publisher_type:archive' --count + # 47 + + fatcat-cli search container 'publisher:"Walter de Gruyter" publisher_type:* !publisher_type:commercial !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "commercial")' -c \ + | python3 ./container_publisher_type.py commercial \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:"springer" publisher_type:* !publisher_type:big5 !publisher_type:archive' --count + # 56 + + fatcat-cli search container 'publisher:"springer" publisher_type:* !publisher_type:big5 !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "big5")' -c \ + | python3 ./container_publisher_type.py big5 \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:"elsevier" publisher_type:* !publisher_type:big5 !publisher_type:archive' --count + # 98 + + fatcat-cli search container 'publisher:"elsevier" publisher_type:* !publisher_type:big5 !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "big5")' -c \ + | python3 ./container_publisher_type.py big5 \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:"wiley" publisher_type:* !publisher_type:big5 !publisher_type:archive' --count + # 37 + + fatcat-cli search container 'publisher:"wiley" publisher_type:* !publisher_type:big5 !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "big5")' -c \ + | python3 ./container_publisher_type.py big5 \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:taylor publisher:francis publisher_type:* !publisher_type:big5 !publisher_type:archive' --count + # 558 + + fatcat-cli search container 'publisher:taylor publisher:francis publisher_type:* !publisher_type:big5 !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "big5")' -c \ + | python3 ./container_publisher_type.py big5 \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + + fatcat-cli search container 'publisher:sage publisher_type:* !publisher_type:big5 !publisher_type:archive' --count + # 28 + + fatcat-cli search container 'publisher:sage publisher_type:* !publisher_type:big5 !publisher_type:archive' --entity-json --limit 300 \ + | jq 'select(.publisher_type != "big5")' -c \ + | python3 ./container_publisher_type.py big5 \ + | fatcat-cli batch update container --description "Update container publisher_type" --auto-accept + +Overall, around a thousand containers updated. Changes to releases will not be +reflected until they are re-indexed. diff --git a/extra/cleanups/file_isiarticles.md b/extra/cleanups/file_isiarticles.md new file mode 100644 index 00000000..3858361c --- /dev/null +++ b/extra/cleanups/file_isiarticles.md @@ -0,0 +1,20 @@ + +The domain isiarticles.com hosts a bunch of partial spam PDFs. + +As a first pass, we can remove these via the domain itself. + +A "blocklist" for this domain has been added to sandcrawler, so they should not +get auto-ingested in the future. + + # 2022-04-20 + fatcat-cli search file domain:isiarticles.com --count + 25067 + +## Prod Cleanup + +See bulk edits log. + +Verify cleanup: + + fatcat-cli search file domain:isiarticles.com '!content_scope:*' --count + 0 diff --git a/extra/container_count_update/update_prod.sh b/extra/container_count_update/update_prod.sh new file mode 100755 index 00000000..766398f2 --- /dev/null +++ b/extra/container_count_update/update_prod.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export CONTAINER_INDEX=fatcat_container_v05_20220110 + +fatcat-cli search container --index-json --limit 0 state:active \ + | jq .ident -r \ + | pv -l \ + > container_idents.tsv + +cat container_idents.tsv \ + | parallel -j10 curl --fail -s 'https://fatcat.wiki/container/{}/stats.json' \ + | jq -c . \ + | pv -l \ + > container_stats.json + +cat container_stats.json \ + | jq '{ ident: .ident, releases_total: .total, preservation_bright: .preservation.bright, preservation_dark: .preservation.dark, preservation_shadows_only: .preservation.shadows_only, preservation_none: .preservation.none }' -c \ + | esbulk -verbose -index $CONTAINER_INDEX -optype update -id ident diff --git a/extra/dblp/.gitignore b/extra/dblp/.gitignore index a04dd76e..60774a12 100644 --- a/extra/dblp/.gitignore +++ b/extra/dblp/.gitignore @@ -4,3 +4,9 @@ series/ Pipfile.lock *.json *.html +*.txt +*.dtd +*.xml +*.xml.gz +*.tsv +*.json.gz diff --git a/extra/dblp/Pipfile b/extra/dblp/Pipfile index dbf86ac0..69705a3a 100644 --- a/extra/dblp/Pipfile +++ b/extra/dblp/Pipfile @@ -5,6 +5,7 @@ name = "pypi" [packages] selectolax = "*" +urlcanon = "*" [dev-packages] diff --git a/extra/dblp/README.md b/extra/dblp/README.md index e6ccce4f..a95f7214 100644 --- a/extra/dblp/README.md +++ b/extra/dblp/README.md @@ -1,14 +1,51 @@ -This file describes hacks used to import dblp container metadata. +This file describes hacks used to import dblp container and release metadata. -As of December 2020 this is part of the dblp release metadata import pipeline: -we must have conference and other non-ISSN containers created before running -the release import. dblp does not publish container-level metadata in a -structured format (eg, in their dumps), so scraping the HTML is unfortunately -necessary. +The container metadata must be processed and imported first, to create +containers for non-ISSN venues. However, dblp only publishes structured +metadata for articles (releases), not venues (containers), so we need to +process the articles, then import the containers, then import the articles. +There is a path that scrapes venue metadata out of dblp.org HTML. -## Quick Bootstrap Commands + +## New Process (2022) + +Usually all of this gets run on a production fatcat instance. It may be +possible to run parts elsewhere, but not confirmed, and would require copying +some set of files around. + + # remove any old/stale files + ./cleanup.sh + + ./prep_container_metadata.sh + +This will take a while to run, after which the container metadata can be +imported, like: + + cd ../../python + pipenv shell + export FATCAT_AUTH_WORKER_DBLP=[...] + ./fatcat_import.py dblp-container --issn-map-file /srv/fatcat/datasets/ISSN-to-ISSN-L.txt --dblp-container-map-file ../extra/dblp/existing_dblp_containers.tsv --dblp-container-map-output ../extra/dblp/all_dblp_containers.tsv ../extra/dblp/dblp_container_meta.json + +Check that counts look sane: + + wc -l existing_dblp_containers.tsv all_dblp_containers.tsv dblp_container_meta.json prefix_list.txt + +Then do release import like: + + cd ../../python + pipenv shell + export FATCAT_AUTH_WORKER_DBLP=[...] + ./fatcat_import.py dblp-release --dblp-container-map-file ../extra/dblp/all_dblp_containers.tsv ../extra/dblp/dblp.xml + +Lastly, to generate sandcrawler ingest requests, from the JSON-dumped partial +release objects:: + + cat dblp_releases_partial.json | pipenv run ./dblp2ingestrequest.py - | pv -l | gzip > dblp_sandcrawler_ingest_requests.json.gz + + +## [OLD] Manual Commands Set up a working directory somewhere: diff --git a/extra/dblp/cleanup.sh b/extra/dblp/cleanup.sh new file mode 100755 index 00000000..52e1a2ea --- /dev/null +++ b/extra/dblp/cleanup.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# run this as 'fatcat' user on a production machine + +rm -f dblp.dtd +rm -f dblp.xml.gz +rm -f dblp.xml +rm -f dblp_releases_partial.json +rm -f prefix_list.txt +rm -f dblp_container_meta.json +rm -f existing_dblp_containers.tsv +rm -f all_dblp_containers.tsv + +rm -rf ./journals/ +rm -rf ./conf/ +rm -rf ./series/ + diff --git a/extra/dblp/dblp2ingestrequest.py b/extra/dblp/dblp2ingestrequest.py new file mode 100755 index 00000000..bdf5575d --- /dev/null +++ b/extra/dblp/dblp2ingestrequest.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Transform a transformed, fatcat-like dblp object (JSON) into zero or more +sandcrawler ingest requests. +""" + +import argparse +import json +import sys + +import urlcanon + +DOMAIN_BLOCKLIST = [ + # we crawl some of these directly via extid; others are just catalogs + "://arxiv.org/", + "://europepmc.org/", + #"://hdl.handle.net/", + "ncbi.nlm.nih.gov/", + "://doi.org/", + "zenodo.org/", + "figshare.com/", + "://d-nb.info/", + "://www.base-search.net/", +] + + +def canon(s): + parsed = urlcanon.parse_url(s) + return str(urlcanon.whatwg(parsed)) + + +def transform(obj): + """ + Transforms from a single object to zero or more ingest requests. + Returns a list of dicts. + """ + + requests = [] + if not obj["ext_ids"].get("dblp"): + return requests + if not obj.get("_dblp_ee_urls"): + return requests + + for url in obj["_dblp_ee_urls"]: + skip = False + for domain in DOMAIN_BLOCKLIST: + if domain in url: + skip = True + if skip: + continue + try: + base_url = canon(url) + except UnicodeEncodeError: + continue + + request = { + "base_url": base_url, + "ingest_type": "pdf", + "link_source": "dblp", + "link_source_id": obj["ext_ids"]["dblp"], + "ingest_request_source": "dblp", + "release_stage": obj.get("release_stage") or None, + "ext_ids": { + "dblp": obj["ext_ids"]["dblp"], + }, + "edit_extra": {}, + } + requests.append(request) + + return requests + + +def run(args): + for l in args.json_file: + if not l.strip(): + continue + row = json.loads(l) + + requests = transform(row) or [] + for r in requests: + print("{}".format(json.dumps(r, sort_keys=True))) + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "json_file", help="dblp transformed JSON file to use", type=argparse.FileType("r") + ) + subparsers = parser.add_subparsers() + + args = parser.parse_args() + + run(args) + + +if __name__ == "__main__": + main() diff --git a/extra/dblp/prep_metadata.sh b/extra/dblp/prep_metadata.sh new file mode 100755 index 00000000..21a50ab0 --- /dev/null +++ b/extra/dblp/prep_metadata.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# run this as 'fatcat' user on a production machine +#export FATCAT_API_HOST="https://api.fatcat.wiki/v0" + +set -e -u -o pipefail + +# ensure deps +#alias fd=fdfind +fd -h > /dev/null +fatcat-cli -h > /dev/null +pipenv -h > /dev/null + +# ensure pipenv is ready +pipenv install +pipenv run true + + +wget -c 'https://dblp.org/xml/dblp.dtd' +wget -c 'https://dblp.org/xml/dblp.xml.gz' + +zcat dblp.xml.gz > dblp.xml + +cd ../../python +pipenv run ./fatcat_import.py dblp-release ../extra/dblp/dblp.xml --dump-json-mode | pv -l > ../extra/dblp/dblp_releases_partial.json + +cd ../extra/dblp/ + +cat dblp_releases_partial.json | jq ._dblp_prefix -r | grep -v ^null | rg '^(journals|conf|series)' | sort -u > prefix_list.txt + +mkdir -p journals +mkdir -p conf +mkdir -p series + +shuf prefix_list.txt | pv -l | parallel -j1 wget -nc -q "https://dblp.org/db/{}/index.html" -O {}.html + +# clean up any failed/empty files, then re-run the above parallel/wget command +find . -empty -type f -delete + +shuf prefix_list.txt | pv -l | parallel -j1 wget -nc -q "https://dblp.org/db/{}/index.html" -O {}.html + +find . -empty -type f -delete + +fd -I html conf/ journals/ series/ | pipenv run ./dblp_html_extract.py | pv -l > dblp_container_meta.json + +fatcat-cli search containers dblp_prefix:* -n 0 --index-json | jq "[.dblp_prefix, .ident] | @tsv" -r | pv -l > existing_dblp_containers.tsv + +cat dblp_releases_partial.json | pipenv run ./dblp2ingestrequest.py - | pv -l | gzip > dblp_sandcrawler_ingest_requests.json.gz diff --git a/extra/sql_dumps/Makefile b/extra/sql_dumps/Makefile new file mode 100644 index 00000000..01607d34 --- /dev/null +++ b/extra/sql_dumps/Makefile @@ -0,0 +1,93 @@ + +SHELL=/bin/bash -euo pipefail +TODAY ?= $(shell date --iso --utc) +DATADIR ?= /srv/fatcat/snapshots/$(TODAY) +DATESLUG ?= $(shell date +%Y-%m-%d.%H%M%S) +DATABASE_URL ?= fatcat_prod + +.PHONY: help +help: ## Print info about all commands + @echo "Commands:" + @echo + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[01;32m%-20s\033[0m %s\n", $$1, $$2}' + +.PHONY: create_datadir +create_datadir: + mkdir -p $(DATADIR)/ + sudo chmod a+rw $(DATADIR)/ + +$(DATADIR)/.IDENTS: + sudo -u postgres DATABASE_URL=$(DATABASE_URL) ./ident_table_snapshot.sh $(DATADIR) + sudo -u postgres mv /tmp/fatcat_ident_*.tsv $(DATADIR) + touch $@ + +$(DATADIR)/release_export_expanded.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_releases_by_work.tsv | sudo -u fatcat ./target/release/fatcat-export releasebywork --expand files,filesets,webcaptures,container -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/creator_export.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_creators.tsv | sudo -u fatcat ./target/release/fatcat-export creator -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/container_export.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_containers.tsv | sudo -u fatcat ./target/release/fatcat-export container -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/file_export.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_files.tsv | sudo -u fatcat ./target/release/fatcat-export file -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/fileset_export.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_filesets.tsv | sudo -u fatcat ./target/release/fatcat-export fileset -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/webcapture_export.json.gz: $(DATADIR)/.IDENTS + cd ../../rust; cat $(DATADIR)/fatcat_ident_webcaptures.tsv | sudo -u fatcat ./target/release/fatcat-export webcapture -j8 | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/abstracts.json.gz: + sudo -u postgres psql $(DATABASE_URL) < dump_abstracts.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/file_hashes.tsv.gz: + sudo -u postgres psql $(DATABASE_URL) < dump_file_hashes.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/release_extid.tsv.gz: + sudo -u postgres psql $(DATABASE_URL) < dump_release_extid.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > $@.wip + mv $@.wip $@ + +$(DATADIR)/.METADATA_EXPORT: $(DATADIR)/.IDENTS $(DATADIR)/release_export_expanded.json.gz $(DATADIR)/creator_export.json.gz $(DATADIR)/container_export.json.gz $(DATADIR)/file_export.json.gz $(DATADIR)/fileset_export.json.gz $(DATADIR)/webcapture_export.json.gz $(DATADIR)/abstracts.json.gz $(DATADIR)/file_hashes.tsv.gz $(DATADIR)/release_extid.tsv.gz ## Dump bulk metadata to disk + touch $@ + +.PHONY: metadata-exports +metadata-exports: create_datadir $(DATADIR)/.METADATA_EXPORT ## Dump bulk metadata to disk + @echo + + +$(DATADIR)/.METADATA_UPLOADED: $(DATADIR)/.METADATA_EXPORT + ia upload --checksum fatcat_bulk_exports_$(TODAY) ia_exports_item_readme.md --remote-name=README.md -m collection:fatcat_snapshots_and_exports -m mediatype:data -m creator:"Internet Archive Web Group" -m date:$(TODAY) -m title:"Fatcat Bulk Metadata Exports ($(TODAY))" + ia upload fatcat_bulk_exports_$(TODAY) $(DATADIR)/*_export.json.gz $(DATADIR)/*_export_expanded.json.gz $(DATADIR)/abstracts.json.gz $(DATADIR)/file_hashes.tsv.gz $(DATADIR)/release_extid.tsv.gz + touch $@ + +.PHONY: upload-metadata-exports +upload-metadata-exports: create_datadir $(DATADIR)/.METADATA_UPLOADED ## Upload bulk metadata exports to archive.org + @echo + +$(DATADIR)/.PUBLIC_DB_DUMP: + sudo -u postgres pg_dump --verbose --format=custom --exclude-table-data=auth_oidc fatcat_prod > $(DATADIR)/fatcat_public_dbdump_${DATESLUG}.pgdump.wip + mv $(DATADIR)/fatcat_public_dbdump_${DATESLUG}.pgdump.wip $(DATADIR)/fatcat_public_dbdump_${DATESLUG}.pgdump + touch $@ + +.PHONY: public-database-snapshot +public-database-snapshot: create_datadir $(DATADIR)/.PUBLIC_DB_DUMP ## Create SQL database snapshot which can be shared publicly + @echo + +$(DATADIR)/.PUBLIC_DB_UPLOADED: $(DATADIR)/.PUBLIC_DB_DUMP + ia upload --checksum fatcat_sqldump_public_$(TODAY) ia_sqldump_item_readme.md --remote-name=README.md -m collection:fatcat_snapshots_and_exports -m mediatype:data -m creator:"Internet Archive Web Group" -m date:$(TODAY) -m title:"Fatcat Public Database Snapshot ($(TODAY))" + ia upload --checksum fatcat_sqldump_public_$(TODAY) $(DATADIR)/fatcat_public_dbdump_*.pgdump + touch $@ + +.PHONY: upload-public-database-snapshot +upload-public-database-snapshot: create_datadir public-database-snapshot $(DATADIR)/.PUBLIC_DB_UPLOADED ## Upload metadata snapshot to archive.org + @echo diff --git a/extra/stats/2022-03-21-prod-stats.json b/extra/stats/2022-03-21-prod-stats.json new file mode 100644 index 00000000..4a82860f --- /dev/null +++ b/extra/stats/2022-03-21-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":5850551,"timestamp":"2022-03-21T22:32:20.050613+00:00"}},"container":{"total":190480},"papers":{"in_kbart":77243855,"in_web":33064706,"in_web_not_kbart":15982780,"is_oa":24345482,"total":126701207},"release":{"refs_total":1269080199,"total":180472435}} diff --git a/extra/stats/2022-03-21-prod-table-sizes.txt b/extra/stats/2022-03-21-prod-table-sizes.txt new file mode 100644 index 00000000..328deec0 --- /dev/null +++ b/extra/stats/2022-03-21-prod-table-sizes.txt @@ -0,0 +1,47 @@ +PostgreSQL 13.5 - wbgrp-svc502.us.archive.org +Size: 707.94G + + table_name | table_size | indexes_size | total_size +---------------------------------------+------------+--------------+------------ + "public"."release_contrib" | 86 GB | 31 GB | 117 GB + "public"."refs_blob" | 114 GB | 2185 MB | 116 GB + "public"."release_rev" | 82 GB | 25 GB | 107 GB + "public"."file_rev" | 35 GB | 29 GB | 64 GB + "public"."release_edit" | 18 GB | 20 GB | 38 GB + "public"."file_rev_url" | 30 GB | 7856 MB | 37 GB + "public"."work_edit" | 17 GB | 19 GB | 36 GB + "public"."abstracts" | 33 GB | 2829 MB | 36 GB + "public"."file_edit" | 17 GB | 15 GB | 33 GB + "public"."release_ident" | 12 GB | 12 GB | 23 GB + "public"."work_ident" | 11 GB | 11 GB | 23 GB + "public"."file_rev_release" | 8709 MB | 10 GB | 19 GB + "public"."file_ident" | 7478 MB | 7579 MB | 15 GB + "public"."work_rev" | 7552 MB | 5238 MB | 12 GB + "public"."release_ref" | 6486 MB | 5199 MB | 11 GB + "public"."release_rev_abstract" | 4718 MB | 5174 MB | 9892 MB + "public"."webcapture_rev_cdx" | 3491 MB | 338 MB | 3829 MB + "public"."creator_edit" | 934 MB | 1042 MB | 1976 MB + "public"."creator_rev" | 928 MB | 730 MB | 1658 MB + "public"."editgroup" | 1224 MB | 252 MB | 1476 MB + "public"."creator_ident" | 631 MB | 647 MB | 1277 MB + "public"."release_rev_extid" | 515 MB | 641 MB | 1157 MB + "public"."changelog" | 351 MB | 297 MB | 648 MB + "public"."container_rev" | 228 MB | 45 MB | 273 MB + "public"."webcapture_edit" | 66 MB | 47 MB | 113 MB + "public"."container_edit" | 59 MB | 54 MB | 113 MB + "public"."webcapture_rev_url" | 54 MB | 20 MB | 74 MB + "public"."webcapture_rev_release" | 20 MB | 35 MB | 54 MB + "public"."webcapture_rev" | 38 MB | 14 MB | 51 MB + "public"."webcapture_ident" | 22 MB | 27 MB | 49 MB + "public"."container_ident" | 13 MB | 20 MB | 33 MB + "public"."editor" | 88 kB | 128 kB | 216 kB + "public"."auth_oidc" | 88 kB | 120 kB | 208 kB + "public"."editgroup_annotation" | 72 kB | 48 kB | 120 kB + "public"."fileset_rev_file" | 48 kB | 32 kB | 80 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB +(41 rows) diff --git a/extra/stats/2022-04-20-prod-stats.json b/extra/stats/2022-04-20-prod-stats.json new file mode 100644 index 00000000..90c673d5 --- /dev/null +++ b/extra/stats/2022-04-20-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":5894575,"timestamp":"2022-04-20T04:19:07.676356+00:00"}},"container":{"total":191000},"papers":{"in_kbart":77487575,"in_web":33633575,"in_web_not_kbart":16531221,"is_oa":24576975,"total":127353365},"release":{"refs_total":1288004961,"total":181877321}} diff --git a/extra/stats/2022-05-15-prod-stats.json b/extra/stats/2022-05-15-prod-stats.json new file mode 100644 index 00000000..37c83f69 --- /dev/null +++ b/extra/stats/2022-05-15-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":5946444,"timestamp":"2022-05-15T21:40:19.956944+00:00"}},"container":{"total":191315},"papers":{"in_kbart":77657266,"in_web":34881556,"in_web_not_kbart":17596263,"is_oa":24739223,"total":127770175},"release":{"refs_total":1303116536,"total":182728162}} diff --git a/extra/stats/2022-07-06-prod-stats.json b/extra/stats/2022-07-06-prod-stats.json new file mode 100644 index 00000000..c93b4e0c --- /dev/null +++ b/extra/stats/2022-07-06-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":6011966,"timestamp":"2022-07-06T23:45:10.513758+00:00"}},"container":{"total":193199},"papers":{"in_kbart":78038966,"in_web":35774136,"in_web_not_kbart":18362815,"is_oa":25221089,"total":128843120},"release":{"refs_total":1335962190,"total":184701189}} diff --git a/extra/stats/2022-07-06-prod-table-sizes.txt b/extra/stats/2022-07-06-prod-table-sizes.txt new file mode 100644 index 00000000..01d205b1 --- /dev/null +++ b/extra/stats/2022-07-06-prod-table-sizes.txt @@ -0,0 +1,48 @@ +PostgreSQL 13.5 - wbgrp-svc502.us.archive.org +Size: 732.62G + + table_name | table_size | indexes_size | total_size +---------------------------------------+------------+--------------+------------ + "public"."release_contrib" | 88 GB | 32 GB | 120 GB + "public"."refs_blob" | 118 GB | 2196 MB | 120 GB + "public"."release_rev" | 85 GB | 25 GB | 110 GB + "public"."file_rev" | 36 GB | 29 GB | 65 GB + "public"."release_edit" | 18 GB | 21 GB | 39 GB + "public"."file_rev_url" | 31 GB | 7999 MB | 39 GB + "public"."abstracts" | 35 GB | 3655 MB | 38 GB + "public"."work_edit" | 17 GB | 19 GB | 37 GB + "public"."file_edit" | 18 GB | 16 GB | 34 GB + "public"."release_ident" | 12 GB | 12 GB | 23 GB + "public"."work_ident" | 12 GB | 11 GB | 23 GB + "public"."file_rev_release" | 8910 MB | 10 GB | 19 GB + "public"."file_ident" | 7704 MB | 7605 MB | 15 GB + "public"."work_rev" | 7741 MB | 5238 MB | 13 GB + "public"."release_ref" | 6714 MB | 5646 MB | 12 GB + "public"."release_rev_abstract" | 5015 MB | 7213 MB | 12 GB + "public"."webcapture_rev_cdx" | 4340 MB | 419 MB | 4758 MB + "public"."creator_edit" | 934 MB | 1042 MB | 1976 MB + "public"."creator_rev" | 928 MB | 730 MB | 1658 MB + "public"."editgroup" | 1282 MB | 256 MB | 1537 MB + "public"."creator_ident" | 631 MB | 647 MB | 1277 MB + "public"."release_rev_extid" | 522 MB | 648 MB | 1170 MB + "public"."changelog" | 378 MB | 301 MB | 679 MB + "public"."container_rev" | 249 MB | 60 MB | 308 MB + "public"."webcapture_edit" | 82 MB | 53 MB | 135 MB + "public"."container_edit" | 63 MB | 69 MB | 132 MB + "public"."webcapture_rev_url" | 65 MB | 22 MB | 87 MB + "public"."webcapture_rev_release" | 24 MB | 35 MB | 59 MB + "public"."webcapture_rev" | 45 MB | 14 MB | 59 MB + "public"."webcapture_ident" | 27 MB | 27 MB | 54 MB + "public"."container_ident" | 13 MB | 20 MB | 34 MB + "public"."auth_oidc" | 104 kB | 160 kB | 264 kB + "public"."editor" | 96 kB | 160 kB | 256 kB + "public"."editgroup_annotation" | 80 kB | 48 kB | 128 kB + "public"."fileset_rev_file" | 88 kB | 32 kB | 120 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB +(41 rows) + diff --git a/extra/stats/2022-07-14-prod-stats.json b/extra/stats/2022-07-14-prod-stats.json new file mode 100644 index 00000000..62d06606 --- /dev/null +++ b/extra/stats/2022-07-14-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":6036957,"timestamp":"2022-07-14T18:53:18.228827+00:00"}},"container":{"total":193300},"papers":{"in_kbart":78102604,"in_web":36247601,"in_web_not_kbart":18551021,"is_oa":25281045,"total":128995907},"release":{"refs_total":1340195856,"total":184966214}} diff --git a/extra/stats/2022-07-14-prod-table-sizes.txt b/extra/stats/2022-07-14-prod-table-sizes.txt new file mode 100644 index 00000000..b4fae69a --- /dev/null +++ b/extra/stats/2022-07-14-prod-table-sizes.txt @@ -0,0 +1,47 @@ +PostgreSQL 13.5 - wbgrp-svc502.us.archive.org +Size: 735.11G + + table_name | table_size | indexes_size | total_size +---------------------------------------+------------+--------------+------------ + "public"."release_contrib" | 88 GB | 32 GB | 121 GB + "public"."refs_blob" | 119 GB | 2200 MB | 121 GB + "public"."release_rev" | 85 GB | 25 GB | 110 GB + "public"."file_rev" | 36 GB | 29 GB | 65 GB + "public"."release_edit" | 18 GB | 21 GB | 39 GB + "public"."file_rev_url" | 31 GB | 8106 MB | 39 GB + "public"."abstracts" | 35 GB | 3671 MB | 39 GB + "public"."work_edit" | 17 GB | 20 GB | 37 GB + "public"."file_edit" | 18 GB | 16 GB | 34 GB + "public"."release_ident" | 12 GB | 12 GB | 23 GB + "public"."work_ident" | 12 GB | 11 GB | 23 GB + "public"."file_rev_release" | 8975 MB | 10 GB | 19 GB + "public"."file_ident" | 7775 MB | 7615 MB | 15 GB + "public"."work_rev" | 7753 MB | 5238 MB | 13 GB + "public"."release_ref" | 6721 MB | 5662 MB | 12 GB + "public"."release_rev_abstract" | 5035 MB | 7250 MB | 12 GB + "public"."webcapture_rev_cdx" | 4341 MB | 419 MB | 4760 MB + "public"."creator_edit" | 934 MB | 1042 MB | 1976 MB + "public"."creator_rev" | 928 MB | 730 MB | 1658 MB + "public"."editgroup" | 1294 MB | 256 MB | 1550 MB + "public"."creator_ident" | 631 MB | 647 MB | 1277 MB + "public"."release_rev_extid" | 524 MB | 649 MB | 1173 MB + "public"."changelog" | 383 MB | 301 MB | 685 MB + "public"."container_rev" | 249 MB | 60 MB | 308 MB + "public"."webcapture_edit" | 82 MB | 53 MB | 135 MB + "public"."container_edit" | 63 MB | 69 MB | 132 MB + "public"."webcapture_rev_url" | 65 MB | 22 MB | 87 MB + "public"."webcapture_rev_release" | 24 MB | 35 MB | 59 MB + "public"."webcapture_rev" | 45 MB | 14 MB | 59 MB + "public"."webcapture_ident" | 27 MB | 27 MB | 54 MB + "public"."container_ident" | 13 MB | 20 MB | 34 MB + "public"."auth_oidc" | 104 kB | 160 kB | 264 kB + "public"."editor" | 96 kB | 160 kB | 256 kB + "public"."editgroup_annotation" | 80 kB | 48 kB | 128 kB + "public"."fileset_rev_file" | 88 kB | 32 kB | 120 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB +(41 rows) diff --git a/extra/stats/2022-07-29-prod-stats.json b/extra/stats/2022-07-29-prod-stats.json new file mode 100644 index 00000000..41d234ea --- /dev/null +++ b/extra/stats/2022-07-29-prod-stats.json @@ -0,0 +1 @@ +{"changelog":{"latest":{"index":6143354,"timestamp":"2022-07-30T01:39:22.900415+00:00"}},"container":{"total":194187},"papers":{"in_kbart":78331069,"in_web":36847123,"in_web_not_kbart":18991886,"is_oa":25579026,"total":130376642},"release":{"refs_total":1350391488,"total":186556315}} diff --git a/extra/stats/2022-07-29-prod-table-sizes.txt b/extra/stats/2022-07-29-prod-table-sizes.txt new file mode 100644 index 00000000..cb85078f --- /dev/null +++ b/extra/stats/2022-07-29-prod-table-sizes.txt @@ -0,0 +1,48 @@ +PostgreSQL 13.5 - wbgrp-svc502.us.archive.org +Size: 748.75G + + table_name | table_size | indexes_size | total_size +---------------------------------------+------------+--------------+------------ + "public"."release_contrib" | 89 GB | 33 GB | 122 GB + "public"."refs_blob" | 119 GB | 2218 MB | 121 GB + "public"."release_rev" | 86 GB | 25 GB | 111 GB + "public"."file_rev" | 36 GB | 29 GB | 65 GB + "public"."release_edit" | 19 GB | 21 GB | 40 GB + "public"."file_rev_url" | 31 GB | 8306 MB | 40 GB + "public"."abstracts" | 35 GB | 3700 MB | 39 GB + "public"."work_edit" | 17 GB | 20 GB | 37 GB + "public"."file_edit" | 19 GB | 16 GB | 34 GB + "public"."release_ident" | 12 GB | 12 GB | 24 GB + "public"."work_ident" | 12 GB | 11 GB | 23 GB + "public"."file_rev_release" | 9031 MB | 10 GB | 19 GB + "public"."file_ident" | 7837 MB | 7624 MB | 15 GB + "public"."work_rev" | 7823 MB | 5243 MB | 13 GB + "public"."release_ref" | 6882 MB | 6039 MB | 13 GB + "public"."release_rev_abstract" | 5100 MB | 7327 MB | 12 GB + "public"."webcapture_rev_cdx" | 8090 MB | 760 MB | 8849 MB + "public"."creator_edit" | 1203 MB | 1919 MB | 3122 MB + "public"."creator_rev" | 1198 MB | 1427 MB | 2624 MB + "public"."creator_ident" | 812 MB | 1258 MB | 2070 MB + "public"."editgroup" | 1325 MB | 261 MB | 1587 MB + "public"."release_rev_extid" | 537 MB | 668 MB | 1204 MB + "public"."changelog" | 395 MB | 307 MB | 702 MB + "public"."container_rev" | 251 MB | 61 MB | 312 MB + "public"."webcapture_edit" | 144 MB | 99 MB | 242 MB + "public"."webcapture_rev_url" | 113 MB | 42 MB | 155 MB + "public"."container_edit" | 63 MB | 71 MB | 135 MB + "public"."webcapture_rev_release" | 40 MB | 70 MB | 110 MB + "public"."webcapture_rev" | 77 MB | 27 MB | 104 MB + "public"."webcapture_ident" | 45 MB | 54 MB | 100 MB + "public"."container_ident" | 13 MB | 21 MB | 34 MB + "public"."auth_oidc" | 104 kB | 160 kB | 264 kB + "public"."editor" | 96 kB | 160 kB | 256 kB + "public"."editgroup_annotation" | 88 kB | 48 kB | 136 kB + "public"."fileset_rev_file" | 88 kB | 32 kB | 120 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB +(41 rows) + diff --git a/extra/stats/2022-09-06-prod-stats.json b/extra/stats/2022-09-06-prod-stats.json new file mode 100644 index 00000000..e7755f9f --- /dev/null +++ b/extra/stats/2022-09-06-prod-stats.json @@ -0,0 +1,146 @@ +<!DOCTYPE html> +<html lang="en" style="position: relative; min-height: 100%; height: auto;"> +<head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="referrer" content="origin-when-cross-origin"> + + <title>Stats | fatcat!</title> + + <link rel="stylesheet" + type="text/css" + href="https://cdn.jsdelivr.net/npm/fomantic-ui@2.8.6/dist/semantic.min.css" + type="text/css" + crossorigin="anonymous"> + <link rel="preload" + href="https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic&subset=latin&display=swap" + as="style"> + <link rel="preload" + href="https://cdn.jsdelivr.net/npm/fomantic-ui@2.8.6/dist/themes/default/assets/fonts/icons.woff2" + as="font" + type="font/woff2" + crossorigin="anonymous"> + <link rel="icon" href="data:,"> + + <style> + .ui.inverted.menu .item:before { background: none; } + + @media only screen and (max-width: 479px) { + .mobile-hide{ display: none !important; } + } + + .field textarea#toml { + font-family: monospace; + } + </style> +</head> +<body style="margin-bottom: 130px;"> + +<header class="ui fixed inverted menu" style="height: 3em;"> + <div class="ui container"> + <a href="/" class="header item"> + <!-- <img class="logo" src="assets/images/logo.png"> --> + fatcat! + </a> + <a href="/about" class="item mobile-hide">About</a> + <a href="https://guide.fatcat.wiki/" class="item mobile-hide">Guide</a> + <a href="/changelog" class="item mobile-hide">Changelog</a> + <div class="right menu"> + <div class="item" style="padding: 0;"> + <form class="" action="/search" method="get" role="search" aria-label="Papers"> + <div class="ui transparent inverted icon input"> + <i class="search icon" style="padding-right: 2em;"></i> + <input type="text" placeholder="Search Papers..." name="q" style="border: 1px solid #777 !important; padding: 5px !important; width: 15em;"> + </div> + </form> + </div> + <div class="ui simple item"> + <a href="/auth/login">Login/Signup</a> + </div> + </div> + </div> +</header> + + +<!-- 4em top margin is "enough" --> +<main class="ui main container" style="margin-top: 6em; margin-bottom: 2em;" > + <div class="ui container text"> + +<h1>Stats</h1> + +You can also fetch these numbers <a href="./stats.json">as JSON</a>. + +<h3>Changelog</h3> + +<p>Latest changelog index is 6205643 (2022-09-06). + +<h3>Entities</h3> + +<table class="ui structured table"> + <tbody> + <tr style="padding-top:0px;"><td rowspan="5" class="active top aligned center aligned"><b>"Papers"</b></td> + <td>Total</td> + <td class="right aligned">131,082,821</td> + <tr> + <td>Fulltext on web</td> + <td class="right aligned">37,792,347</td> + <tr> + <td>"Gold" Open Access</td> + <td class="right aligned">25,854,622</td> + <tr> + <td>In a Keepers/KBART archive</td> + <td class="right aligned">78,632,782</td> + <tr> + <td>On web, not in Keepers</td> + <td class="right aligned">19,770,890</td> + + <tr style="padding-top:0px;"><td rowspan="1" class="active top aligned center aligned"><b>Releases</b></td> + <td>Total</td> + <td class="right aligned">188,266,679</td> + + <tr style="padding-top:0px;"><td rowspan="1" class="active top aligned center aligned"><b>Containers</b></td> + <td>Total</td> + <td class="right aligned">194,685</td> + </tbody> +</table> + +<br> +<i>"Papers" are journal articles and conference proceedings, a subset of Releases</i> + + </div> +</main> + + +<footer class="ui inverted vertical footer segment" style="margin-top: 2em; padding-top: 2em; padding-bottom:2em; position: absolute; bottom: 0px; width: 100%;"> + <div class="ui center aligned container"> + <div class="ui horizontal inverted small divided link list"> + <a class="item" href="/">fatcat!</a> + <a class="item" href="/about">About</a> + <a class="item" href="https://guide.fatcat.wiki/sources.html">Sources</a> + <a class="item" href="https://stats.uptimerobot.com/GM9YNSrB0">Status</a> + <a class="item" href="https://guide.fatcat.wiki/bulk_exports.html">Bulk Exports</a> + <a class="item" href="https://github.com/internetarchive/fatcat/">Code</a> + <a class="item" href="https://github.com/internetarchive/fatcat/tree/v0.5.1-109-g5ecf72cb"><code>v0.5.1-109-g5ecf72cb</code></a> + </div> + </div> +</footer> + +<script + src="https://cdn.jsdelivr.net/npm/jquery@3.3.1/dist/jquery.min.js" + integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" + crossorigin="anonymous"> +</script> +<script + src="https://cdn.jsdelivr.net/npm/fomantic-ui@2.8.6/dist/semantic.min.js" + integrity="sha256-9H3HWYnPJ2bEHgkOrw+48KheOqYzTvJd1hbeU9sEDFk=" + crossorigin="anonymous"> +</script> +<script> + window.goatcounter = { + /* title leaks search query, so don't report it */ + title: "", + }; + </script> + <script data-goatcounter="/goatcounter/count" async src="/goatcounter/count.js"></script> +</body> +</html>
\ No newline at end of file diff --git a/extra/stats/2022-09-06-table-sizes.txt b/extra/stats/2022-09-06-table-sizes.txt new file mode 100644 index 00000000..ddbd6842 --- /dev/null +++ b/extra/stats/2022-09-06-table-sizes.txt @@ -0,0 +1,48 @@ + +PostgreSQL 13.5 - wbgrp-svc502.us.archive.org +Size: 760.02G + + table_name | table_size | indexes_size | total_size +---------------------------------------+------------+--------------+------------ + "public"."release_contrib" | 90 GB | 34 GB | 124 GB + "public"."refs_blob" | 121 GB | 2295 MB | 123 GB + "public"."release_rev" | 87 GB | 26 GB | 112 GB + "public"."file_rev" | 36 GB | 30 GB | 66 GB + "public"."file_rev_url" | 32 GB | 8778 MB | 40 GB + "public"."release_edit" | 19 GB | 21 GB | 40 GB + "public"."abstracts" | 36 GB | 3726 MB | 40 GB + "public"."work_edit" | 18 GB | 20 GB | 38 GB + "public"."file_edit" | 19 GB | 16 GB | 35 GB + "public"."release_ident" | 12 GB | 12 GB | 24 GB + "public"."work_ident" | 12 GB | 12 GB | 23 GB + "public"."file_rev_release" | 9100 MB | 10 GB | 19 GB + "public"."file_ident" | 7914 MB | 7647 MB | 15 GB + "public"."release_ref" | 7012 MB | 6486 MB | 13 GB + "public"."work_rev" | 7900 MB | 5280 MB | 13 GB + "public"."release_rev_abstract" | 5217 MB | 7395 MB | 12 GB + "public"."webcapture_rev_cdx" | 9173 MB | 862 MB | 10035 MB + "public"."creator_edit" | 1203 MB | 1919 MB | 3122 MB + "public"."creator_rev" | 1198 MB | 1427 MB | 2624 MB + "public"."creator_ident" | 812 MB | 1258 MB | 2070 MB + "public"."editgroup" | 1347 MB | 272 MB | 1620 MB + "public"."release_rev_extid" | 540 MB | 672 MB | 1212 MB + "public"."changelog" | 406 MB | 318 MB | 724 MB + "public"."container_rev" | 251 MB | 62 MB | 313 MB + "public"."webcapture_edit" | 168 MB | 105 MB | 273 MB + "public"."webcapture_rev_url" | 133 MB | 44 MB | 177 MB + "public"."container_edit" | 63 MB | 72 MB | 135 MB + "public"."webcapture_rev_release" | 48 MB | 71 MB | 119 MB + "public"."webcapture_rev" | 91 MB | 27 MB | 118 MB + "public"."webcapture_ident" | 54 MB | 55 MB | 108 MB + "public"."container_ident" | 13 MB | 21 MB | 34 MB + "public"."editor" | 104 kB | 168 kB | 272 kB + "public"."auth_oidc" | 104 kB | 160 kB | 264 kB + "public"."editgroup_annotation" | 88 kB | 48 kB | 136 kB + "public"."fileset_rev_file" | 88 kB | 32 kB | 120 kB + "public"."fileset_edit" | 16 kB | 48 kB | 64 kB + "public"."fileset_rev_url" | 16 kB | 32 kB | 48 kB + "public"."fileset_rev_release" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_ident" | 8192 bytes | 32 kB | 40 kB + "public"."fileset_rev" | 16 kB | 16 kB | 32 kB + "public"."__diesel_schema_migrations" | 8192 bytes | 16 kB | 24 kB +(41 rows) diff --git a/guide/src/entity_file.md b/guide/src/entity_file.md index 84d9eac4..6a11e945 100644 --- a/guide/src/entity_file.md +++ b/guide/src/entity_file.md @@ -26,6 +26,10 @@ many articles), and that a release will often have multiple files (differing only by watermarks, or different digitizations of the same printed work, or variant MIME/media types of the same published work). +- `extra` (object with string keys): additional metadata about this file + - `path`: filename, with optional path prefix. path must be "relative", not + "absolute", and should use UNIX-style forward slashes, not Windows-style + backward slashes #### URL `rel` Vocabulary diff --git a/guide/src/entity_fileset.md b/guide/src/entity_fileset.md index 6083a09d..8a9ea832 100644 --- a/guide/src/entity_fileset.md +++ b/guide/src/entity_fileset.md @@ -10,16 +10,17 @@ - `sha1` (string): SHA-1 hash in lower-case hex - `sha256` (string): SHA-256 hash in lower-case hex - `mimetype` (string): Content type in MIME type schema - - `extra` (object): any extra metadata about this specific file - - `original_url`: live web canonical URL to download this file (optional) - - `webarchive_url`: web archive capture of this file (optional) - - `platform_id`: platform-specific identifier for this file + - `extra` (object): any extra metadata about this specific file. all are + optional + - `original_url`: live web canonical URL to download this file + - `webarchive_url`: web archive capture of this file - `urls`: An array of "typed" URLs. Order is not meaningful, and may not be - preserved. + preserved. These are URLs for the entire fileset, not individual files. - `url` (string, required): Eg: "https://example.edu/~frau/prcding.pdf". - `rel` (string, required): - Eg: "webarchive". + Eg: "archive-base", "webarchive". + - `release_ids` (array of string identifiers): references to `release` entities - `content_scope` (string): for situations where the fileset does not simply contain the full representation of a work (eg, all files in dataset, for a @@ -27,12 +28,18 @@ vocabulary as File entity. - `extra` (object with string keys): additional metadata about this group of files, including upstream platform-specific metadata and identifiers + - `platform_id`: platform-specific identifier for this fileset #### URL `rel` types -- `repository`: URL of a live-web landing page or other location where content can be - found. May not be machine-reachable. -- `webarchive`: web archive version of `repository` +Any ending in "-base" implies that a file path (from the manifest) can be +appended to the "base" URL to get a file download URL. Any "bundle" implies a +direct link to an archive or "bundle" (like `.zip` or `.tar`) which contains +all the files in this fileset + +- `repository` or `platform` or `web`: URL of a live-web landing page or other + location where content can be found. May or may not be machine-reachable. +- `webarchive`: web archive version of `repository` landing page - `repository-bundle`: direct URL to a live-web "archive" file, such as `.zip`, which contains all of the individual files in this fileset - `webarchive-bundle`: web archive version of `repository-bundle` diff --git a/notes/2022-08-12_changelog_gap.md b/notes/2022-08-12_changelog_gap.md new file mode 100644 index 00000000..48572973 --- /dev/null +++ b/notes/2022-08-12_changelog_gap.md @@ -0,0 +1,41 @@ + +On 2022-08-11, realized that we had a "gap" in the changelog: after a VM +reboot, the postgresql primary key sequence for the 'changelog' table had been +incremented, but rows were not inserted (transaction hadn't finished). + +This was a known potential problem (naively relying on the sequence to +increment with no gaps). + +As a work-around, implemented a simple "gap filler" which will create +empty/dummy editgroups and changelog entries. + +This gap extends from 6153703 to 6153721, so just a couple dozen entries. The +fixup command was: + + ./target/release/fatcat-doctor backfill-changelog-gap 6153702 6153721 + Inserted changelog: 6153703 + Inserted changelog: 6153704 + Inserted changelog: 6153705 + Inserted changelog: 6153706 + Inserted changelog: 6153707 + Inserted changelog: 6153708 + Inserted changelog: 6153709 + Inserted changelog: 6153710 + Inserted changelog: 6153711 + Inserted changelog: 6153712 + Inserted changelog: 6153713 + Inserted changelog: 6153714 + Inserted changelog: 6153715 + Inserted changelog: 6153716 + Inserted changelog: 6153717 + Inserted changelog: 6153718 + Inserted changelog: 6153719 + Inserted changelog: 6153720 + Inserted changelog: 6153721 + +After that the changelog worker was happy: + + Aug 13 02:41:59 wbgrp-svc502.us.archive.org fatcat-worker[386037]: Most recent changelog index in Kafka seems to be 6153702 + Aug 13 02:41:59 wbgrp-svc502.us.archive.org fatcat-worker[386037]: Fetching changelogs from 6153703 through 6158547 + Aug 13 02:43:12 wbgrp-svc502.us.archive.org fatcat-worker[386037]: Sleeping 5.0 seconds... + diff --git a/notes/merge_releases_examples.txt b/notes/merge_releases_examples.txt index ca65705e..a2f3e297 100644 --- a/notes/merge_releases_examples.txt +++ b/notes/merge_releases_examples.txt @@ -19,3 +19,6 @@ https://fatcat.wiki/release/search?q=NeuroTrends+Visualization 45 versions across two figshare works +https://fatcat.wiki/container/hqhtzu2pufgulc5pdw2tfx55e4 + + container has several DOAJ/DOI duplicates diff --git a/python/fatcat_import.py b/python/fatcat_import.py index f502d4ed..2896577e 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -25,6 +25,7 @@ from fatcat_tools.importers import ( FilesetImporter, GrobidMetadataImporter, IngestFileResultImporter, + IngestFilesetFileResultImporter, IngestFilesetResultImporter, IngestWebResultImporter, JalcImporter, @@ -232,6 +233,30 @@ def run_ingest_fileset(args: argparse.Namespace) -> None: JsonLinePusher(ifri, args.json_file).run() +def run_ingest_fileset_file(args: argparse.Namespace) -> None: + ifri = IngestFilesetFileResultImporter( + args.api, + editgroup_description=args.editgroup_description_override, + skip_source_allowlist=args.skip_source_allowlist, + do_updates=args.do_updates, + default_link_rel=args.default_link_rel, + edit_batch_size=args.batch_size, + ) + if args.kafka_mode: + KafkaJsonPusher( + ifri, + args.kafka_hosts, + args.kafka_env, + "ingest-fileset-results", + "fatcat-{}-ingest-fileset-result".format(args.kafka_env), + kafka_namespace="sandcrawler", + consume_batch_size=args.batch_size, + force_flush=True, + ).run() + else: + JsonLinePusher(ifri, args.json_file).run() + + def run_savepapernow_file(args: argparse.Namespace) -> None: ifri = SavePaperNowFileImporter( args.api, @@ -750,6 +775,39 @@ def main() -> None: help="default URL rel for matches (eg, 'publisher', 'web')", ) + sub_ingest_fileset_file = subparsers.add_parser( + "ingest-fileset-file-results", + help="add/update file entities linked to releases based on sandcrawler dataset/fileset ingest results", + ) + sub_ingest_fileset_file.set_defaults( + func=run_ingest_fileset_file, + auth_var="FATCAT_AUTH_WORKER_CRAWL", + ) + sub_ingest_fileset_file.add_argument( + "json_file", + help="ingest_fileset JSON file to import from", + default=sys.stdin, + type=argparse.FileType("r"), + ) + sub_ingest_fileset_file.add_argument( + "--skip-source-allowlist", + action="store_true", + help="don't filter import based on request source allowlist", + ) + sub_ingest_fileset_file.add_argument( + "--kafka-mode", action="store_true", help="consume from kafka topic (not stdin)" + ) + sub_ingest_fileset_file.add_argument( + "--do-updates", + action="store_true", + help="update pre-existing fileset entities if new match (instead of skipping)", + ) + sub_ingest_fileset_file.add_argument( + "--default-link-rel", + default="fileset", + help="default URL rel for matches (eg, 'publisher', 'web')", + ) + sub_savepapernow_file = subparsers.add_parser( "savepapernow-file-results", help="add file entities crawled due to async Save Paper Now request", diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py index 654be2e9..e13ab552 100644 --- a/python/fatcat_tools/importers/__init__.py +++ b/python/fatcat_tools/importers/__init__.py @@ -37,6 +37,7 @@ from .fileset_generic import FilesetImporter from .grobid_metadata import GrobidMetadataImporter from .ingest import ( IngestFileResultImporter, + IngestFilesetFileResultImporter, IngestFilesetResultImporter, IngestWebResultImporter, SavePaperNowFileImporter, diff --git a/python/fatcat_tools/importers/arxiv.py b/python/fatcat_tools/importers/arxiv.py index dd2c2284..d7d3ed97 100644 --- a/python/fatcat_tools/importers/arxiv.py +++ b/python/fatcat_tools/importers/arxiv.py @@ -18,6 +18,9 @@ latex2text = LatexNodes2Text() def latex_to_text(raw: str) -> str: + # hack: handle a single special mangled title + if raw.startswith("%CRTFASTGEEPWR"): + return raw.strip() try: return latex2text.latex_to_text(raw).strip() except AttributeError: @@ -26,6 +29,11 @@ def latex_to_text(raw: str) -> str: return raw.strip() +def test_latex_to_text() -> None: + s = "%CRTFASTGEEPWR: a SAS macro for power of the generalized estimating equations of multi-period cluster randomized trials with application to stepped wedge designs" + assert latex_to_text(s) == s + + def parse_arxiv_authors(raw: str) -> List[str]: if not raw: return [] diff --git a/python/fatcat_tools/importers/chocula.py b/python/fatcat_tools/importers/chocula.py index 8c410d3e..38802bcb 100644 --- a/python/fatcat_tools/importers/chocula.py +++ b/python/fatcat_tools/importers/chocula.py @@ -136,6 +136,9 @@ class ChoculaImporter(EntityImporter): do_update = True if ce.extra.get("webarchive_urls") and not ce.extra.get("webarchive_urls", []): do_update = True + if ce.extra.get("publisher_type") and not ce.extra.get("publisher_type"): + # many older containers were missing this metadata + do_update = True for k in ("kbart", "ia", "doaj"): # always update these fields if not equal (chocula override) if ce.extra.get(k) and ce.extra[k] != existing.extra.get(k): diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index cd51a24c..2136d1da 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -916,3 +916,14 @@ def make_kafka_consumer( ) print("Consuming from kafka topic {}, group {}".format(topic_name, group)) return consumer + + +def filesets_very_similar(a: FilesetEntity, b: FilesetEntity) -> bool: + """ + This helper method checks if two Fileset entities are effectively equivalent: same set of files with comparable hashes. + + Uses a set() of SHA1 hashes to test for equivalence. + """ + a_hashes = set([f.sha1 for f in a.manifest]) + b_hashes = set([f.sha1 for f in b.manifest]) + return a_hashes == b_hashes diff --git a/python/fatcat_tools/importers/dblp_release.py b/python/fatcat_tools/importers/dblp_release.py index 5f78ca3a..b97c3976 100644 --- a/python/fatcat_tools/importers/dblp_release.py +++ b/python/fatcat_tools/importers/dblp_release.py @@ -542,18 +542,21 @@ class DblpReleaseImporter(EntityImporter): Used only in JSON dump mode, with the intent of transforming into sandcrawler ingest requests. """ - EXTID_PATTERNS = [ + SKIP_PATTERNS = [ "://doi.acm.org/", "://doi.ieeecomputersociety.org/", "doi.org/10.", "wikidata.org/entity/Q", "://arxiv.org/abs/", + "://hdl.handle.net/", + "://d-nb.info/", + "://www.base-search.net/", ] urls = [] for ee in xml_elem.find_all("ee"): url = ee.text skip = False - for pattern in EXTID_PATTERNS: + for pattern in SKIP_PATTERNS: if pattern in url: skip = True break diff --git a/python/fatcat_tools/importers/doaj_article.py b/python/fatcat_tools/importers/doaj_article.py index 8f5e7acf..64c05773 100644 --- a/python/fatcat_tools/importers/doaj_article.py +++ b/python/fatcat_tools/importers/doaj_article.py @@ -100,6 +100,10 @@ class DoajArticleImporter(EntityImporter): container_name = None break + if not container_id: + self.counts["skip-no-container"] += 1 + return None + volume = clean_str(bibjson["journal"].get("volume")) # NOTE: this schema seems to use "number" as "issue number" issue = clean_str(bibjson["journal"].get("number")) diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py index 4f1cc3c4..c1fed31f 100644 --- a/python/fatcat_tools/importers/ingest.py +++ b/python/fatcat_tools/importers/ingest.py @@ -11,7 +11,7 @@ from fatcat_openapi_client import ( WebcaptureEntity, ) -from .common import EntityImporter, make_rel_url +from .common import EntityImporter, filesets_very_similar, make_rel_url class IngestFileResultImporter(EntityImporter): @@ -260,6 +260,16 @@ class IngestFileResultImporter(EntityImporter): edit_extra["grobid_status_code"] = row["grobid"]["status_code"] edit_extra["grobid_version"] = row["grobid"].get("grobid_version") + # fileset/platform metadata + if row.get("ingest_strategy"): + edit_extra["ingest_strategy"] = row["ingest_strategy"] + if row.get("platform_domain"): + edit_extra["platform_domain"] = row["platform_domain"] + if row.get("platform_name"): + edit_extra["platform_name"] = row["platform_name"] + if row.get("platform_id"): + edit_extra["platform_id"] = row["platform_id"] + return edit_extra def parse_record(self, row: Dict[str, Any]) -> FileEntity: @@ -518,7 +528,6 @@ class IngestWebResultImporter(IngestFileResultImporter): ) edit_extra = self.parse_edit_extra(row) - if edit_extra: wc.edit_extra = edit_extra return wc @@ -675,9 +684,9 @@ class IngestFilesetResultImporter(IngestFileResultImporter): return True def parse_fileset_urls(self, row: Dict[str, Any]) -> List[FilesetUrl]: - if not row.get("strategy"): + if not row.get("ingest_strategy"): return [] - strategy = row["strategy"] + strategy = row["ingest_strategy"] urls = [] if strategy == "archiveorg-fileset" and row.get("archiveorg_item_name"): urls.append( @@ -686,17 +695,14 @@ class IngestFilesetResultImporter(IngestFileResultImporter): rel="archive-base", ) ) - if row["strategy"].startswith("web-") and row.get("platform_base_url"): + if strategy.startswith("web-") and row.get("platform_base_url"): urls.append( fatcat_openapi_client.FilesetUrl( url=f"https://web.archive.org/web/{row['web_base_url_dt']}/{row['web_base_url']}", rel="webarchive-base", ) ) - # TODO: repository-base - # TODO: web-base - - if row["strategy"] == "archiveorg-fileset-bundle" and row.get("archiveorg_item_name"): + if strategy == "archiveorg-fileset-bundle" and row.get("archiveorg_item_name"): urls.append( fatcat_openapi_client.FilesetUrl( url=f"https://archive.org/download/{row['archiveorg_item_name']}/{row['archiveorg_bundle_path']}", @@ -704,7 +710,7 @@ class IngestFilesetResultImporter(IngestFileResultImporter): ) ) - if row["strategy"] == "web-fileset-bundle" and row.get("platform_bundle_url"): + if strategy == "web-fileset-bundle" and row.get("platform_bundle_url"): urls.append( fatcat_openapi_client.FilesetUrl( url=f"https://web.archive.org/web/{row['web_bundle_url_dt']}/{row['web_bundle_url']}", @@ -727,6 +733,15 @@ class IngestFilesetResultImporter(IngestFileResultImporter): rel="repository-base", ) ) + elif row.get("terminal"): + # fallback generic web URL + urls.append( + fatcat_openapi_client.FilesetUrl( + url=row["terminal"]["terminal_url"], + rel="web", + ) + ) + return urls def parse_record(self, row: Dict[str, Any]) -> FilesetEntity: @@ -748,12 +763,6 @@ class IngestFilesetResultImporter(IngestFileResultImporter): return None entity_extra: Dict[str, Any] = dict() - edit_extra = self.parse_edit_extra(row) - edit_extra["ingest_strategy"] = row["ingest_strategy"] - if row.get("platform"): - edit_extra["platform"] = row["platform"] - if row.get("platform_id"): - edit_extra["platform_id"] = row["platform_id"] entity_urls = self.parse_fileset_urls(row) if not entity_urls: @@ -770,33 +779,33 @@ class IngestFilesetResultImporter(IngestFileResultImporter): fsf = fatcat_openapi_client.FilesetFile( path=ingest_file["path"], size=ingest_file["size"], - md5=ingest_file["md5"], - sha1=ingest_file["sha1"], + md5=ingest_file.get("md5"), + sha1=ingest_file.get("sha1"), sha256=ingest_file.get("sha256"), - extra=dict( - mimetype=ingest_file["mimetype"], - ), + mimetype=ingest_file.get("mimetype"), + extra=dict(), ) - if not (fsf.md5 and fsf.sha1 and fsf.path and fsf.size): + if not (fsf.md5 and fsf.sha1 and fsf.path and fsf.size and fsf.mimetype): self.counts["skip-partial-file-info"] += 1 return None if ingest_file.get("platform_url"): - # XXX: should we include this? fsf.extra["original_url"] = ingest_file["platform_url"] if ingest_file.get("terminal_url") and ingest_file.get("terminal_dt"): fsf.extra[ "wayback_url" ] = f"https://web.archive.org/web/{ingest_file['terminal_dt']}/{ingest_file['terminal_url']}" + if not fsf.extra: + fsf.extra = None manifest.append(fsf) fe = fatcat_openapi_client.FilesetEntity( manifest=manifest, urls=entity_urls, release_ids=[release_ident], + extra=entity_extra or None, ) - if entity_extra: - fe.extra = entity_extra + edit_extra = self.parse_edit_extra(row) if edit_extra: fe.edit_extra = edit_extra return fe @@ -805,26 +814,29 @@ class IngestFilesetResultImporter(IngestFileResultImporter): # check for existing edits-in-progress with same URL for other in self._entity_queue: - # XXX: how to duplicate check? - if other.original_url == fse.original_url: + if filesets_very_similar(other, fse): self.counts["skip-in-queue"] += 1 + self.counts["skip"] += 1 return False # lookup sha1, or create new entity (TODO: API doesn't support this yet) # existing = None # NOTE: in lieu of existing checks (by lookup), only allow one fileset per release - release = self.api.get_release(fse.release_ids[0], expand="filesets") - if release.filesets: - # XXX: how to duplicate check filesets? + if not self.bezerk_mode: + release = self.api.get_release(fse.release_ids[0], expand="filesets") + # check if this is an existing match, or just a similar hit - for other in release.filesets: - if fse.original_url == other.original_url: - # TODO: compare very similar timestamps of same time (different formats) + for other in release.filesets or []: + if filesets_very_similar(other, fse): self.counts["exists"] += 1 return False - self.counts["skip-release-has-fileset"] += 1 - return False + + # for now, being conservative and just skipping if release has any other fileset + if release.filesets: + self.counts["skip-release-has-fileset"] += 1 + self.counts["skip"] += 1 + return False return True @@ -849,6 +861,197 @@ class IngestFilesetResultImporter(IngestFileResultImporter): ) +class IngestFilesetFileResultImporter(IngestFileResultImporter): + """ + Variant of IngestFileResultImporter for processing dataset (Fileset) ingest + results, which resulted in a single file, into File entities. + """ + + def __init__(self, api: ApiClient, **kwargs) -> None: + + eg_desc = ( + kwargs.pop("editgroup_description", None) + or "Single files crawled from web using sandcrawler ingest tool, in dataset mode" + ) + eg_extra = kwargs.pop("editgroup_extra", dict()) + eg_extra["agent"] = eg_extra.get( + "agent", "fatcat_tools.IngestFilesetFileResultImporter" + ) + kwargs["do_updates"] = False + super().__init__(api, editgroup_description=eg_desc, editgroup_extra=eg_extra, **kwargs) + self.max_file_count = 300 + + def want_fileset(self, row: Dict[str, Any]) -> bool: + + manifest: Optional[List[Any]] = row.get("manifest") + if not manifest or len(manifest) == 0: + self.counts["skip-empty-manifest"] += 1 + return False + + if len(manifest) > 1: + self.counts["skip-multiple-files"] += 1 + return False + + assert len(manifest) == 1 + return True + + def want(self, row: Dict[str, Any]) -> bool: + + if not self.want_ingest(row): + return False + + if row.get("status") != "success-file": + self.counts["skip-status"] += 1 + return False + + # fileset-specific filters + if row["request"].get("ingest_type") not in [ + "dataset", + ]: + self.counts["skip-ingest-type"] += 1 + return False + + if not self.want_fileset(row): + return False + + return True + + def parse_fileset_urls(self, row: Dict[str, Any]) -> List[FilesetUrl]: + if not row.get("ingest_strategy"): + return [] + strategy = row["ingest_strategy"] + urls = [] + # XXX + if strategy == "archiveorg-fileset" and row.get("archiveorg_item_name"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=f"https://archive.org/download/{row['archiveorg_item_name']}/", + rel="archive-base", + ) + ) + if strategy.startswith("web-") and row.get("platform_base_url"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=f"https://web.archive.org/web/{row['web_base_url_dt']}/{row['web_base_url']}", + rel="webarchive-base", + ) + ) + if strategy == "archiveorg-fileset-bundle" and row.get("archiveorg_item_name"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=f"https://archive.org/download/{row['archiveorg_item_name']}/{row['archiveorg_bundle_path']}", + rel="archive-bundle", + ) + ) + + if strategy == "web-fileset-bundle" and row.get("platform_bundle_url"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=f"https://web.archive.org/web/{row['web_bundle_url_dt']}/{row['web_bundle_url']}", + rel="webarchive-bundle", + ) + ) + + # add any additional / platform URLs here + if row.get("platform_bundle_url"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=row["platform_bundle_url"], + rel="repository-bundle", + ) + ) + if row.get("platform_base_url"): + urls.append( + fatcat_openapi_client.FilesetUrl( + url=row["platform_bundle_url"], + rel="repository-base", + ) + ) + elif row.get("terminal"): + # fallback generic web URL + urls.append( + fatcat_openapi_client.FilesetUrl( + url=row["terminal"]["terminal_url"], + rel="web", + ) + ) + + return urls + + def parse_record(self, row: Dict[str, Any]) -> FileEntity: + + request = row["request"] + + # double check that want() filtered request correctly + if request.get("ingest_type") not in [ + "dataset", + ]: + self.counts["skip-ingest-type"] += 1 + return None + + # identify release by fatcat ident, or extid lookup + release_ident = self.parse_ingest_release_ident(row) + + if not release_ident: + self.counts["skip-release-not-found"] += 1 + return None + + assert row["file_count"] == len(row["manifest"]) == 1 + file_meta = row["manifest"][0] + # print(file_meta) + assert file_meta["status"] == "success" + + # add file-level access URLs + entity_urls = [] + if file_meta.get("platform_url"): + entity_urls.append(FileUrl(rel="web", url=file_meta["platform_url"])) + if file_meta.get("terminal_url") and file_meta.get("terminal_dt"): + entity_urls.append( + FileUrl( + rel="webarchive", + url=f"https://web.archive.org/web/{file_meta['terminal_dt']}/{file_meta['terminal_url']}", + ) + ) + if row["ingest_strategy"] == "archiveorg-file": + entity_urls.append( + FileUrl( + rel="archive", + url=f"https://archive.org/download/{row['archiveorg_item_name']}/{file_meta['path']}", + ) + ) + + if not entity_urls: + self.counts["skip-no-access-url"] += 1 + return None + + entity_extra: Dict[str, Any] = dict() + entity_extra["path"] = file_meta["path"] + + # this is to work around a bug in old sandcrawler ingest code + if file_meta["md5"] == file_meta["sha1"]: + self.counts["skip-bad-hashes"] += 1 + return None + + fe = FileEntity( + md5=file_meta["md5"], + sha1=file_meta["sha1"], + sha256=file_meta["sha256"], + size=file_meta["size"], + mimetype=file_meta["mimetype"], + release_ids=[release_ident], + urls=entity_urls, + extra=entity_extra or None, + ) + if not (fe.md5 and fe.sha1 and fe.sha256 and (fe.size is not None) and fe.mimetype): + self.counts["skip-partial-file-info"] += 1 + return None + + edit_extra = self.parse_edit_extra(row) + if edit_extra: + fe.edit_extra = edit_extra + return fe + + class SavePaperNowFilesetImporter(IngestFilesetResultImporter): """ Like SavePaperNowFileImporter, but for fileset/dataset ingest. diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py index cbf9e9bf..1a19a651 100644 --- a/python/fatcat_tools/transforms/ingest.py +++ b/python/fatcat_tools/transforms/ingest.py @@ -85,6 +85,14 @@ def release_ingest_request( url = "https://doi.org/{}".format(release.ext_ids.doi.lower()) link_source = "doi" link_source_id = release.ext_ids.doi.lower() + elif release.ext_ids.doaj: + url = "https://doaj.org/article/{}".format(release.ext_ids.doaj.lower()) + link_source = "doaj" + link_source_id = release.ext_ids.doaj.lower() + elif release.ext_ids.hdl: + url = "https://hdl.handle.net/{}".format(release.ext_ids.hdl.lower()) + link_source = "hdl" + link_source_id = release.ext_ids.hdl.lower() if not url: return None diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 1f6602d4..4ceeac49 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -3,6 +3,7 @@ import os from typing import Any, Callable, Dict, List, Optional import citeproc_styles +import sentry_sdk from fatcat_openapi_client import EditgroupAnnotation from fatcat_openapi_client.rest import ApiException, ApiValueError from flask import ( @@ -1450,7 +1451,14 @@ def release_bibtex(ident: str) -> AnyResponse: entity = api.get_release(ident) except ApiException as ae: raise ae - csl = release_to_csl(entity) + try: + csl = release_to_csl(entity) + except ValueError as e: + # "handle" the missing author/surname path, so we don't get exception + # reports about it. these are not linked to, only show up from bots. + sentry_sdk.set_level("warning") + sentry_sdk.capture_exception(e) + abort(400, e) bibtex = citeproc_csl(csl, "bibtex") return Response(bibtex, mimetype="text/plain") @@ -1468,7 +1476,14 @@ def release_citeproc(ident: str) -> AnyResponse: entity = api.get_release(ident) except ApiException as ae: raise ae - csl = release_to_csl(entity) + try: + csl = release_to_csl(entity) + except ValueError as e: + # "handle" the missing author/surname path, so we don't get exception + # reports about it. these are not linked to, only show up from bots. + sentry_sdk.set_level("warning") + sentry_sdk.capture_exception(e) + abort(400, e) try: cite = citeproc_csl(csl, style, is_html) except citeproc_styles.StyleNotFoundError as e: diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 913d6696..f85f6c89 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -636,7 +636,8 @@ def get_elastic_preservation_by_year(query: ReleaseQuery) -> List[Dict[str, Any] "biblio", ], ) - search = search.filter("term", container_id=query.container_id) + if query.container_id: + search = search.filter("term", container_id=query.container_id) if query.exclude_stubs: search = search.query( "bool", diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 5f8f6e0a..4386e698 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -75,6 +75,14 @@ <a href="{{ u }}">{{ u }}</a><br> {% endfor %} </code> + {% elif key in ("manifest", "abstracts", "refs", "releases", "files", "webcaptures", "filesets", "urls", "archive_urls", "cdx", "contribs") and value and value is iterable and value is not string %} + <tr><td class="three wide right aligned"><code>{{ key }}[]</code></td> + <td class="seven wide"> + <code> + {% for m in value %} + {% if m != None %}{{ m }}{% endif %}<br> + {% endfor %} + </code> {% else %} <tr><td class="three wide right aligned"><code>{{ key }}</code></td> <td class="seven wide"><code>{% if value != None %}{{ value }}{% endif %}</code> diff --git a/python/fatcat_web/templates/fileset_view.html b/python/fatcat_web/templates/fileset_view.html index 4e8bfbad..6f3f802e 100644 --- a/python/fatcat_web/templates/fileset_view.html +++ b/python/fatcat_web/templates/fileset_view.html @@ -3,6 +3,21 @@ {% import "entity_macros.html" as entity_macros %} {% extends "entity_base.html" %} +{% macro file_mimetype_icon(mimetype) -%} + {%- if not mimetype -%}file outline + {%- elif mimetype in ["application/pdf"] -%}file pdf outline + {%- elif mimetype in ["application/x-hdf"] -%}database + {%- elif mimetype in ["text/csv", "text/tab-separated-values"] -%}table + {%- elif mimetype in ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] -%}table + {%- elif mimetype in ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] -%}file alternate outline + {%- elif mimetype in ["application/zip", "application/gzip", "application/x-tar", "application/x-rar", "application/x-bzip2"] -%}file archive outline + {%- elif mimetype in ["application/octet-stream"] -%}save outline + {%- elif mimetype.startswith("text/") -%}file alternate outline + {%- elif mimetype.startswith("image/") -%}file image outline + {%- elif mimetype.startswith("video/") -%}film + {%- else -%}file outline{% endif %} +{%- endmacro %} + {% block entity_main %} <div class="ui stackable mobile reversed grid centered"> @@ -23,26 +38,64 @@ No known public URL, mirror, or archive for this File Set. {% endif %} +{% set fileset_vars = namespace(archiveorg_base=None, webarchiveorg_base=None) %} +{% for u in (entity.urls or []) %} + {% if u.rel in ["archive-base"] %} + {% set fileset_vars.archiveorg_base = u.url %} + {% elif u.rel in ["webarchive-base"] %} + {% set fileset_vars.webarchiveorg_base = u.url %} + {% endif %} +{% endfor %} + <h3>File Manifest ({{ fileset.manifest|count }})</h3> {% if fileset.manifest %} - <div class="ui celled list"> +<div style="overflow-x: auto;"> +<table class="ui compact single line table"> + <thead> + <tr> + <th class="collapsing"></th> + <th>Path</th> + <th>Type</th> + <th>Size</th> + <th class="collapsing"></th> + </tr> + </thead> + <tbody> {% for file in fileset.manifest %} - <div class="item"> - <div class="content"><code> - <div class="header"> - <i class="file outline icon"></i> {{ file.path }} ({{ file.size|filesizeformat }}{% if file.extra.mimetype %}, {{ file.extra.mimetype }}{% endif %}) - </div> - <div style="color: #666; margin-left: 1em;"><small> - {% if file.md5 %} md5:{{ file.md5 }}<br>{% endif %} - {% if file.sha1 %} sha1:{{ file.sha1 }}<br>{% endif %} - {% if file.sha256 %}sha256:{{ file.sha256 }}<br>{% endif %} + <tr> + <td><i class="{{ file_mimetype_icon(file.mimetype) }} icon" style="font-size: x-large;"></i></td> + <td> + <b><code>{{ file.path }}</code></b> + <br><div style="color: #666; margin-left: 0em;"><small> + {% if file.sha1 %} sha1:{{ file.sha1 }}<br> + {% elif file.sha256 %}sha256:{{ file.sha256 }}<br> + {% elif file.md5 %} md5:{{ file.md5 }}<br> + {% endif %} </small></div> - </code></div> - </div> + </td> + <td>{% if file.mimetype %}<code style="font-size: smaller;">{{ file.mimetype }}</code>{% endif %}</td> + <td>{{ file.size|filesizeformat }}</td> + <td> + {% if fileset_vars.archiveorg_base %} + <a href="{{ fileset_vars.archiveorg_base }}/{{ file.path }}">[archive.org]</a><br> + {% endif %} + {% if fileset_vars.webarchiveorg_base %} + <a href="{{ fileset_vars.webarchiveorg_base }}/{{ file.path }}">[web.archive.org]</a><br> + {% endif %} + {% if file.extra and file.extra.original_url %} + <a href="{{ file.extra.original_url }}">[platform]</a><br> + {% endif %} + {% if file.extra and file.extra.webarchive_url%} + <a href="{{ file.extra.webarchive_url }}">[web.archive.org]</a><br> + {% endif %} + </td> + </tr> {% endfor %} - </div> + </tbody> +</table> +</div> {% else %} -This File Set is empty (contains no files). +<p>This File Set is empty (contains no files). {% endif %} diff --git a/python/fatcat_web/templates/release_view.html b/python/fatcat_web/templates/release_view.html index f2e3f670..53ac3703 100644 --- a/python/fatcat_web/templates/release_view.html +++ b/python/fatcat_web/templates/release_view.html @@ -131,8 +131,9 @@ {% endif %} {% if entity.state == 'active' %} + +{% if entity.files != [] and entity.files != None %} <h3>Archived Files and Locations</h3> -{% if entity.files != [] %} <table class="ui compact fixed table"> <tbody> {% for file in entity.files %} @@ -151,15 +152,9 @@ {% endfor %} </tbody> </table> -{% else %} -<p>There are no accessible files associated with this release. You could check -<a href="/work/{{ release.work_id }}">other releases for this work</a> for an -accessible version. -{% endif %} {% endif %} -{% if entity.state == 'active' %} -{% if entity.filesets != [] %} +{% if entity.filesets != [] and entity.filesets != None %} <h3>File Sets</h3> <table class="ui compact fixed table"> <tbody> @@ -181,10 +176,8 @@ accessible version. </tbody> </table> {% endif %} -{% endif %} -{% if entity.state == 'active' %} -{% if entity.webcaptures != [] %} +{% if entity.webcaptures != [] and entity.webcaptures != None %} <h3>Web Captures</h3> <table class="ui single line compact fixed table"> <tbody> @@ -203,9 +196,17 @@ accessible version. </tbody> </table> {% endif %} + +{% if not (entity.files or entity.filesets or entity.webcaptures) %} +<h3>Archived Content</h3> +<p>There are no accessible files associated with this release. You could check +<a href="/work/{{ release.work_id }}">other releases for this work</a> for an +accessible version. +{% endif %} + {% endif %} -</div> +</div> <div class="column" style="flex: 0 0 24em;"> @@ -213,7 +214,7 @@ accessible version. <a href="{{ entity._es.ia_pdf_url }}" class="ui fluid huge black button" style="text-decoration: underline;"> <i class="file pdf outline icon"></i>Read Archived PDF </a> -{% elif entity.state == 'active' and entity.webcaptures != [] and entity.webcaptures[0].archive_urls != [] and entity.webcaptures[0].archive_urls[0].rel == "wayback" %} +{% elif entity.state == 'active' and entity.webcaptures != None and entity.webcaptures != [] and entity.webcaptures[0].archive_urls != [] and entity.webcaptures[0].archive_urls[0].rel == "wayback" %} <a href="{{ entity.webcaptures[0].archive_urls[0].url }}{{ entity.webcaptures[0]._wayback_suffix }}" class="ui fluid huge black button" style="text-decoration: underline;"> <i class="university icon"></i>Visit Web Archive </a> @@ -354,7 +355,7 @@ accessible version. #} {% endif %} {% if release.container._es.any_kbart == True %} - <i class="icon check green"></i> In <a href="https://keepers.issn.org/?q=api/search&search[]=MUST=allissn={{ release.container.issnl }}&search[]=MUST_EXIST=keepers">Keepers Registery</a><br> + <i class="icon check green"></i> In <a href="https://keepers.issn.org/?q=api/search&search[]=MUST=allissn={{ release.container.issnl }}&search[]=MUST_EXIST=keepers">Keepers Registry</a><br> {% elif release.container._es.any_kbart == False %} <i class="icon times grey"></i> Not in <a href="https://keepers.issn.org/?q=api/search&search[]=MUST=allissn={{ release.container.issnl }}&search[]=MUST_EXIST=keepers">Keepers Registry</a><br> {% endif %} {% endif %} diff --git a/python/tests/files/ISSN-to-ISSN-L.snip.txt b/python/tests/files/ISSN-to-ISSN-L.snip.txt index 2569c443..96cfb4c0 100644 --- a/python/tests/files/ISSN-to-ISSN-L.snip.txt +++ b/python/tests/files/ISSN-to-ISSN-L.snip.txt @@ -9,3 +9,4 @@ ISSN ISSN-L 0000-0108 0002-0108 0000-0140 0002-0140 1877-3273 1877-3273 +1234-5678 1234-5678 diff --git a/python/tests/files/example_doaj_articles.json b/python/tests/files/example_doaj_articles.json index 018a4800..5e018176 100644 --- a/python/tests/files/example_doaj_articles.json +++ b/python/tests/files/example_doaj_articles.json @@ -1,5 +1,5 @@ -{"last_updated":"2020-02-04T14:11:44Z","bibjson":{"identifier":[{"id":"0264-1275","type":"pissn"},{"id":"10.1016/j.matdes.2016.06.110","type":"DOI"}],"journal":{"volume":"108","number":"","country":"GB","license":[{"open_access":true,"title":"CC BY-NC-ND","type":"CC BY-NC-ND","url":"https://www.elsevier.com/journals/materials-and-design/0264-1275/open-access-journal"}],"issns":["0264-1275","1873-4197"],"publisher":"Elsevier","language":["EN"],"title":"Materials & Design"},"month":"10","end_page":"617","year":"2016","start_page":"608","subject":[{"code":"TA401-492","scheme":"LCC","term":"Materials of engineering and construction. Mechanics of materials"}],"author":[{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Xinfeng Li"},{"affiliation":"Department of Geosciences, Center for Materials by Design, State University of New York, Stony Brook, NY 11794-2100, USA","name":"Jin Zhang"},{"affiliation":"School of Chemical Engineering & Technology, China University of Mining and Technology, Xuzhou 221116, China","name":"Yanfei Wang"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Sicong Shen"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China; Corresponding author.","name":"Xiaolong Song"}],"link":[{"type":"fulltext","url":"http://www.sciencedirect.com/science/article/pii/S0264127516308723"}],"abstract":"The tensile properties and fracture behavior of PH 13-8 Mo steel after subjected to pre-charged hydrogen were investigated by slow strain rate tensile tests. The results suggest that hydrogen slightly increases yield strength, while decreases tensile strength. The susceptibility to hydrogen embrittlement of specimens aged at 650 °C firstly reduces and then increases as the aging time increases, reaching the lowest value at aging time 4 h. This is dominantly attributed to the highest content of austenite. Moreover, hydrogen-induced crack nucleation sites initiate from lath, packet and prior austenite grain boundaries. Crack propagation passes through lath boundaries and walks along packet, prior austenite grain boundaries. Scanning electron microscopy result indicates that hydrogen-charged specimens show quasi-cleavage fracture and intergranular fracture in annular brittle zone while dimple fracture is observed in hydrogen-free specimens. Keywords: Hydrogen embrittlement, PH 13-8 Mo steel, Aging time, Fracture behavior","title":"Effect of hydrogen on tensile properties and fracture behavior of PH 13-8 Mo steel"},"created_date":"2019-06-05T05:25:15Z","id":"e58f08a11ecb495ead55a44ad4f89808"} -{"last_updated":"2020-02-04T08:06:42Z","bibjson":{"identifier":[{"id":"2072-6694","type":"eissn"},{"id":"10.3390/cancers9080107","type":"doi"}],"journal":{"volume":"9","number":"8","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/cancers/about"}],"issns":["2072-6694"],"publisher":"MDPI AG","language":["EN"],"title":"Cancers"},"month":"8","keywords":["ALK rearrangement, lung cancer, biology, immunohistochemistry, FISH, molecular biology."],"year":"2017","start_page":"107","subject":[{"code":"RC254-282","scheme":"LCC","term":"Neoplasms. Tumors. Oncology. Including cancer and carcinogens"}],"author":[{"affiliation":"Laboratory of Clinical and Experimental Pathology, Pasteur Hospital, 30 avenue de la voie romaine, 06001 Nice cedex 01, France","name":"Paul Hofman"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/2072-6694/9/8/107"}],"abstract":"Patients with advanced-stage non-small cell lung carcinoma (NSCLC) harboring an ALK rearrangement, detected from a tissue sample, can benefit from targeted ALK inhibitor treatment. Several increasingly effective ALK inhibitors are now available for treatment of patients. However, despite an initial favorable response to treatment, in most cases relapse or progression occurs due to resistance mechanisms mainly caused by mutations in the tyrosine kinase domain of ALK. The detection of an ALK rearrangement is pivotal and can be done using different methods, which have variable sensitivity and specificity depending, in particular, on the quality and quantity of the patient’s sample. This review will first highlight briefly some information regarding the pathobiology of an ALK rearrangement and the epidemiology of patients harboring this genomic alteration. The different methods used to detect an ALK rearrangement as well as their advantages and disadvantages will then be examined and algorithms proposed for detection in daily routine practice.","title":"ALK in Non-Small Cell Lung Cancer (NSCLC) Pathobiology, Epidemiology, Detection from Tumor Tissue and Algorithm Diagnosis in a Daily Practice"},"admin":{"seal":true},"created_date":"2018-10-26T07:49:34Z","id":"937c7aa790e048d4ae5f53a2ad71f0dc"} -{"last_updated":"2020-02-04T13:43:13Z","bibjson":{"identifier":[{"id":"1178-2013","type":"pissn"}],"end_page":"818","keywords":["bioconjugation","biosurfactant","cancer therapy","folic acid receptor","graphene quantum dots","theranostic tool"],"year":"2019","subject":[{"code":"R5-920","scheme":"LCC","term":"Medicine (General)"}],"author":[{"name":"Bansal S"},{"name":"Singh J"},{"name":"Kumari U"},{"name":"Kaur IP"},{"name":"Barnwal RP"},{"name":"Kumar R"},{"name":"Singh S"},{"name":"Singh G"},{"name":"Chatterjee M"}],"link":[{"content_type":"html","type":"fulltext","url":"https://www.dovepress.com/development-of-biosurfactant-based-graphene-quantum-dot-conjugate-as-a-peer-reviewed-article-IJN"}],"abstract":"Smriti Bansal,1 Joga Singh,2 Uma Kumari,3 Indu Pal Kaur,2 Ravi Pratap Barnwal,4 Ravinder Kumar,3 Suman Singh,5 Gurpal Singh,2 Mary Chatterjee1 1Biotechnology Engineering, University Institute of Engineering & Technology, Panjab University, Chandigarh, India; 2Department of Pharmaceutical Sciences, University Institute of Pharmaceutical Sciences, Panjab University, Chandigarh, India; 3Department of Zoology, Panjab University, Chandigarh, India; 4Department of Biophysics, Panjab University, Chandigarh, India; 5Department of Agronomics, Central Scientific Instruments Organisation, Chandigarh, India Background: Biosurfactants are amphipathic molecules of microbial origin that reduce surface and interfacial tension at gas–liquid–solid interfaces. Earlier, the biosurfactant was isolated and characterized in our laboratory from Candida parapsilosis. The property of the biosurfactant is further explored in this study by using quantum dots (QDs) as nanocarrier.Materials and methods: Graphene quantum dots (GQDs) were synthesized by bottom-up approach through pyrolysis of citric acid. GQDs were conjugated with both biosurfactant and folic acid (FA) using carbodiimide chemistry. The prepared GQD bioconjugate was studied for diagnostic and therapeutic effects against cancer cells.Results and discussion: Photoluminescence quantum yield (QY) of plain GQDs was measured as 12.8%. QY for biosurfactant conjugated GQDs and FA-biosurfactant conjugated GQDs was measured as 10.4% and 9.02%, respectively, and it was sufficient for targeting cancer cells. MTT assay showed that more than 90% of cells remained viable at concentration of 1 mg/mL, hence GQDs seemed to be non-toxic to cells. Biosurfactant conjugated GQDs caused 50% reduction in cellular viability within 24 hours. FA conjugation further increased the specificity of bioconjugated GQDs toward tumor cells, which is clearly evident from the drug internalization studies using confocal laser scanning microscopy. A higher amount of drug uptake was observed when bioconjugated GQDs were decorated with FA.Conclusion: The ability of GQD bioconjugate could be used as a theranostic tool for cancer. It is foreseen that in near future cancer can be detected and/or treated at an early stage by utilizing biosurfactant conjugated GQDs. Therefore, the proposed study would provide a stepping stone to improve the life of cancer patients. Keywords: bioconjugation, nanomedicine, nanocarrier, cancer therapy, folic acid receptor, graphene quantum dots","title":"Development of biosurfactant-based graphene quantum dot conjugate as a novel and fluorescent theranostic tool for cancer","journal":{"volume":"Volume 14","country":"GB","license":[{"open_access":true,"title":"CC BY-NC","type":"CC BY-NC","url":"https://www.dovepress.com/author_guidelines.php?content_id=695"}],"issns":["1176-9114","1178-2013"],"publisher":"Dove Medical Press","language":["EN"],"title":"International Journal of Nanomedicine"},"month":"1","start_page":"809"},"created_date":"2019-01-29T18:43:40Z","id":"e0173c80437f4fb88ec4e02e453e13b0"} -{"last_updated":"2020-02-04T09:46:14Z","bibjson":{"identifier":[{"id":"1424-8220","type":"eissn"},{"id":"10.3390/s18124467","type":"doi"}],"journal":{"volume":"18","number":"12","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/sensors/about"}],"issns":["1424-8220"],"publisher":"MDPI AG","language":["EN"],"title":"Sensors"},"month":"12","keywords":["multilayer sea ice temperature","low temperature","design","performance analysis"],"year":"2018","start_page":"4467","subject":[{"code":"TP1-1185","scheme":"LCC","term":"Chemical technology"}],"author":[{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Guangyu Zuo"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yinke Dou"},{"affiliation":"College of Water Resources Science and Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Xiaomin Chang"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yan Chen"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Chunyan Ma"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/1424-8220/18/12/4467"}],"abstract":"Temperature profiles of sea ice have been recorded more than a few decades. However, few high-precision temperature sensors can complete the observation of temperature profile of sea ice, especially in extreme environments. At present, the most widely used sea ice observation instruments can reach an accuracy of sea ice temperature measurement of 0.1 °C. In this study, a multilayer sea ice temperature sensor is developed with temperature measurement accuracy from −0.0047 °C to 0.0059 °C. The sensor system composition, structure of the thermistor string, and work mode are analyzed. The performance of the sensor system is evaluated from −50 °C to 30 °C. The temperature dependence of the constant current source, the amplification circuit, and the analog-to-digital converter (ADC) circuit are comprehensive tested and quantified. A temperature correction algorithm is designed to correct any deviation in the sensor system. A sea-ice thickness discrimination algorithm is proposed in charge of determining the thickness of sea ice automatically. The sensor system was field tested in Wuliangsuhai, Yellow River on 31 January 2018 and the second reservoir of Fen River, Yellow River on 30 January 2018. The integral practicality of this sensor system is identified and examined. The multilayer sea ice temperature sensor will provide good temperature results of sea ice and maintain stable performance in the low ambient temperature.","title":"Design and Performance Analysis of a Multilayer Sea Ice Temperature Sensor Used in Polar Region"},"admin":{"seal":true},"created_date":"2018-12-18T08:13:29Z","id":"152f83d12b9f477696e681684ba696e7"} -{"last_updated":"2020-06-02T23:02:32Z","bibjson":{"identifier":[{"id":"10.123/abc","type":"doi"},{"id":"2076-3417","type":"eissn"}],"journal":{"volume":"10","number":"3872","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/about/openaccess"}],"issns":["2076-3417"],"publisher":"MDPI AG","language":["EN"],"title":"Applied Sciences"},"month":"06","keywords":["Smart parking systems","survey","vehicle routing problem","vehicle detection techniques","routing algorithms"],"year":"2020","start_page":"3872","subject":[{"code":"T","scheme":"LCC","term":"Technology"},{"code":"TA1-2040","scheme":"LCC","term":"Engineering (General). Civil engineering (General)"},{"code":"QH301-705.5","scheme":"LCC","term":"Biology (General)"},{"code":"QC1-999","scheme":"LCC","term":"Physics"},{"code":"QD1-999","scheme":"LCC","term":"Chemistry"}],"author":[{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Mathias Gabriel Diaz Ogás"},{"affiliation":"Institute of Informatics and Applications. University of Girona, 17003 Girona, Spain","name":"Ramon Fabregat"},{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Silvana Aciar"}],"link":[{"content_type":"text/html","type":"fulltext","url":"https://www.mdpi.com/2076-3417/10/11/3872"}],"abstract":"The large number of vehicles constantly seeking access to congested areas in cities means that finding a public parking place is often difficult and causes problems for drivers and citizens alike. In this context, strategies that guide vehicles from one point to another, looking for the most optimal path, are needed. Most contributions in the literature are routing strategies that take into account different criteria to select the optimal route required to find a parking space. This paper aims to identify the types of smart parking systems (SPS) that are available today, as well as investigate the kinds of vehicle detection techniques (VDT) they have and the algorithms or other methods they employ, in order to analyze where the development of these systems is at today. To do this, a survey of 274 publications from January 2012 to December 2019 was conducted. The survey considered four principal features: SPS types reported in the literature, the kinds of VDT used in these SPS, the algorithms or methods they implement, and the stage of development at which they are. Based on a search and extraction of results methodology, this work was able to effectively obtain the current state of the research area. In addition, the exhaustive study of the studies analyzed allowed for a discussion to be established concerning the main difficulties, as well as the gaps and open problems detected for the SPS. The results shown in this study may provide a base for future research on the subject.","title":"Survey of Smart Parking Systems"},"admin":{"seal":true},"id":"9cf511bab39445ba9745feb43d7493dd","created_date":"2020-06-03T00:02:28Z"} +{"last_updated":"2020-02-04T14:11:44Z","bibjson":{"identifier":[{"id":"0264-1275","type":"pissn"},{"id":"10.1016/j.matdes.2016.06.110","type":"DOI"}],"journal":{"volume":"108","number":"","country":"GB","license":[{"open_access":true,"title":"CC BY-NC-ND","type":"CC BY-NC-ND","url":"https://www.elsevier.com/journals/materials-and-design/0264-1275/open-access-journal"}],"issns":["1234-5678","0264-1275","1873-4197"],"publisher":"Elsevier","language":["EN"],"title":"Materials & Design"},"month":"10","end_page":"617","year":"2016","start_page":"608","subject":[{"code":"TA401-492","scheme":"LCC","term":"Materials of engineering and construction. Mechanics of materials"}],"author":[{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Xinfeng Li"},{"affiliation":"Department of Geosciences, Center for Materials by Design, State University of New York, Stony Brook, NY 11794-2100, USA","name":"Jin Zhang"},{"affiliation":"School of Chemical Engineering & Technology, China University of Mining and Technology, Xuzhou 221116, China","name":"Yanfei Wang"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Sicong Shen"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China; Corresponding author.","name":"Xiaolong Song"}],"link":[{"type":"fulltext","url":"http://www.sciencedirect.com/science/article/pii/S0264127516308723"}],"abstract":"The tensile properties and fracture behavior of PH 13-8 Mo steel after subjected to pre-charged hydrogen were investigated by slow strain rate tensile tests. The results suggest that hydrogen slightly increases yield strength, while decreases tensile strength. The susceptibility to hydrogen embrittlement of specimens aged at 650 °C firstly reduces and then increases as the aging time increases, reaching the lowest value at aging time 4 h. This is dominantly attributed to the highest content of austenite. Moreover, hydrogen-induced crack nucleation sites initiate from lath, packet and prior austenite grain boundaries. Crack propagation passes through lath boundaries and walks along packet, prior austenite grain boundaries. Scanning electron microscopy result indicates that hydrogen-charged specimens show quasi-cleavage fracture and intergranular fracture in annular brittle zone while dimple fracture is observed in hydrogen-free specimens. Keywords: Hydrogen embrittlement, PH 13-8 Mo steel, Aging time, Fracture behavior","title":"Effect of hydrogen on tensile properties and fracture behavior of PH 13-8 Mo steel"},"created_date":"2019-06-05T05:25:15Z","id":"e58f08a11ecb495ead55a44ad4f89808"} +{"last_updated":"2020-02-04T08:06:42Z","bibjson":{"identifier":[{"id":"2072-6694","type":"eissn"},{"id":"10.3390/cancers9080107","type":"doi"}],"journal":{"volume":"9","number":"8","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/cancers/about"}],"issns":["1234-5678","2072-6694"],"publisher":"MDPI AG","language":["EN"],"title":"Cancers"},"month":"8","keywords":["ALK rearrangement, lung cancer, biology, immunohistochemistry, FISH, molecular biology."],"year":"2017","start_page":"107","subject":[{"code":"RC254-282","scheme":"LCC","term":"Neoplasms. Tumors. Oncology. Including cancer and carcinogens"}],"author":[{"affiliation":"Laboratory of Clinical and Experimental Pathology, Pasteur Hospital, 30 avenue de la voie romaine, 06001 Nice cedex 01, France","name":"Paul Hofman"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/2072-6694/9/8/107"}],"abstract":"Patients with advanced-stage non-small cell lung carcinoma (NSCLC) harboring an ALK rearrangement, detected from a tissue sample, can benefit from targeted ALK inhibitor treatment. Several increasingly effective ALK inhibitors are now available for treatment of patients. However, despite an initial favorable response to treatment, in most cases relapse or progression occurs due to resistance mechanisms mainly caused by mutations in the tyrosine kinase domain of ALK. The detection of an ALK rearrangement is pivotal and can be done using different methods, which have variable sensitivity and specificity depending, in particular, on the quality and quantity of the patient’s sample. This review will first highlight briefly some information regarding the pathobiology of an ALK rearrangement and the epidemiology of patients harboring this genomic alteration. The different methods used to detect an ALK rearrangement as well as their advantages and disadvantages will then be examined and algorithms proposed for detection in daily routine practice.","title":"ALK in Non-Small Cell Lung Cancer (NSCLC) Pathobiology, Epidemiology, Detection from Tumor Tissue and Algorithm Diagnosis in a Daily Practice"},"admin":{"seal":true},"created_date":"2018-10-26T07:49:34Z","id":"937c7aa790e048d4ae5f53a2ad71f0dc"} +{"last_updated":"2020-02-04T13:43:13Z","bibjson":{"identifier":[{"id":"1178-2013","type":"pissn"}],"end_page":"818","keywords":["bioconjugation","biosurfactant","cancer therapy","folic acid receptor","graphene quantum dots","theranostic tool"],"year":"2019","subject":[{"code":"R5-920","scheme":"LCC","term":"Medicine (General)"}],"author":[{"name":"Bansal S"},{"name":"Singh J"},{"name":"Kumari U"},{"name":"Kaur IP"},{"name":"Barnwal RP"},{"name":"Kumar R"},{"name":"Singh S"},{"name":"Singh G"},{"name":"Chatterjee M"}],"link":[{"content_type":"html","type":"fulltext","url":"https://www.dovepress.com/development-of-biosurfactant-based-graphene-quantum-dot-conjugate-as-a-peer-reviewed-article-IJN"}],"abstract":"Smriti Bansal,1 Joga Singh,2 Uma Kumari,3 Indu Pal Kaur,2 Ravi Pratap Barnwal,4 Ravinder Kumar,3 Suman Singh,5 Gurpal Singh,2 Mary Chatterjee1 1Biotechnology Engineering, University Institute of Engineering & Technology, Panjab University, Chandigarh, India; 2Department of Pharmaceutical Sciences, University Institute of Pharmaceutical Sciences, Panjab University, Chandigarh, India; 3Department of Zoology, Panjab University, Chandigarh, India; 4Department of Biophysics, Panjab University, Chandigarh, India; 5Department of Agronomics, Central Scientific Instruments Organisation, Chandigarh, India Background: Biosurfactants are amphipathic molecules of microbial origin that reduce surface and interfacial tension at gas–liquid–solid interfaces. Earlier, the biosurfactant was isolated and characterized in our laboratory from Candida parapsilosis. The property of the biosurfactant is further explored in this study by using quantum dots (QDs) as nanocarrier.Materials and methods: Graphene quantum dots (GQDs) were synthesized by bottom-up approach through pyrolysis of citric acid. GQDs were conjugated with both biosurfactant and folic acid (FA) using carbodiimide chemistry. The prepared GQD bioconjugate was studied for diagnostic and therapeutic effects against cancer cells.Results and discussion: Photoluminescence quantum yield (QY) of plain GQDs was measured as 12.8%. QY for biosurfactant conjugated GQDs and FA-biosurfactant conjugated GQDs was measured as 10.4% and 9.02%, respectively, and it was sufficient for targeting cancer cells. MTT assay showed that more than 90% of cells remained viable at concentration of 1 mg/mL, hence GQDs seemed to be non-toxic to cells. Biosurfactant conjugated GQDs caused 50% reduction in cellular viability within 24 hours. FA conjugation further increased the specificity of bioconjugated GQDs toward tumor cells, which is clearly evident from the drug internalization studies using confocal laser scanning microscopy. A higher amount of drug uptake was observed when bioconjugated GQDs were decorated with FA.Conclusion: The ability of GQD bioconjugate could be used as a theranostic tool for cancer. It is foreseen that in near future cancer can be detected and/or treated at an early stage by utilizing biosurfactant conjugated GQDs. Therefore, the proposed study would provide a stepping stone to improve the life of cancer patients. Keywords: bioconjugation, nanomedicine, nanocarrier, cancer therapy, folic acid receptor, graphene quantum dots","title":"Development of biosurfactant-based graphene quantum dot conjugate as a novel and fluorescent theranostic tool for cancer","journal":{"volume":"Volume 14","country":"GB","license":[{"open_access":true,"title":"CC BY-NC","type":"CC BY-NC","url":"https://www.dovepress.com/author_guidelines.php?content_id=695"}],"issns":["1234-5678","1176-9114","1178-2013"],"publisher":"Dove Medical Press","language":["EN"],"title":"International Journal of Nanomedicine"},"month":"1","start_page":"809"},"created_date":"2019-01-29T18:43:40Z","id":"e0173c80437f4fb88ec4e02e453e13b0"} +{"last_updated":"2020-02-04T09:46:14Z","bibjson":{"identifier":[{"id":"1424-8220","type":"eissn"},{"id":"10.3390/s18124467","type":"doi"}],"journal":{"volume":"18","number":"12","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/sensors/about"}],"issns":["1234-5678","1424-8220"],"publisher":"MDPI AG","language":["EN"],"title":"Sensors"},"month":"12","keywords":["multilayer sea ice temperature","low temperature","design","performance analysis"],"year":"2018","start_page":"4467","subject":[{"code":"TP1-1185","scheme":"LCC","term":"Chemical technology"}],"author":[{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Guangyu Zuo"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yinke Dou"},{"affiliation":"College of Water Resources Science and Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Xiaomin Chang"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yan Chen"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Chunyan Ma"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/1424-8220/18/12/4467"}],"abstract":"Temperature profiles of sea ice have been recorded more than a few decades. However, few high-precision temperature sensors can complete the observation of temperature profile of sea ice, especially in extreme environments. At present, the most widely used sea ice observation instruments can reach an accuracy of sea ice temperature measurement of 0.1 °C. In this study, a multilayer sea ice temperature sensor is developed with temperature measurement accuracy from −0.0047 °C to 0.0059 °C. The sensor system composition, structure of the thermistor string, and work mode are analyzed. The performance of the sensor system is evaluated from −50 °C to 30 °C. The temperature dependence of the constant current source, the amplification circuit, and the analog-to-digital converter (ADC) circuit are comprehensive tested and quantified. A temperature correction algorithm is designed to correct any deviation in the sensor system. A sea-ice thickness discrimination algorithm is proposed in charge of determining the thickness of sea ice automatically. The sensor system was field tested in Wuliangsuhai, Yellow River on 31 January 2018 and the second reservoir of Fen River, Yellow River on 30 January 2018. The integral practicality of this sensor system is identified and examined. The multilayer sea ice temperature sensor will provide good temperature results of sea ice and maintain stable performance in the low ambient temperature.","title":"Design and Performance Analysis of a Multilayer Sea Ice Temperature Sensor Used in Polar Region"},"admin":{"seal":true},"created_date":"2018-12-18T08:13:29Z","id":"152f83d12b9f477696e681684ba696e7"} +{"last_updated":"2020-06-02T23:02:32Z","bibjson":{"identifier":[{"id":"10.123/abc","type":"doi"},{"id":"2076-3417","type":"eissn"}],"journal":{"volume":"10","number":"3872","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/about/openaccess"}],"issns":["1234-5678","2076-3417"],"publisher":"MDPI AG","language":["EN"],"title":"Applied Sciences"},"month":"06","keywords":["Smart parking systems","survey","vehicle routing problem","vehicle detection techniques","routing algorithms"],"year":"2020","start_page":"3872","subject":[{"code":"T","scheme":"LCC","term":"Technology"},{"code":"TA1-2040","scheme":"LCC","term":"Engineering (General). Civil engineering (General)"},{"code":"QH301-705.5","scheme":"LCC","term":"Biology (General)"},{"code":"QC1-999","scheme":"LCC","term":"Physics"},{"code":"QD1-999","scheme":"LCC","term":"Chemistry"}],"author":[{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Mathias Gabriel Diaz Ogás"},{"affiliation":"Institute of Informatics and Applications. University of Girona, 17003 Girona, Spain","name":"Ramon Fabregat"},{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Silvana Aciar"}],"link":[{"content_type":"text/html","type":"fulltext","url":"https://www.mdpi.com/2076-3417/10/11/3872"}],"abstract":"The large number of vehicles constantly seeking access to congested areas in cities means that finding a public parking place is often difficult and causes problems for drivers and citizens alike. In this context, strategies that guide vehicles from one point to another, looking for the most optimal path, are needed. Most contributions in the literature are routing strategies that take into account different criteria to select the optimal route required to find a parking space. This paper aims to identify the types of smart parking systems (SPS) that are available today, as well as investigate the kinds of vehicle detection techniques (VDT) they have and the algorithms or other methods they employ, in order to analyze where the development of these systems is at today. To do this, a survey of 274 publications from January 2012 to December 2019 was conducted. The survey considered four principal features: SPS types reported in the literature, the kinds of VDT used in these SPS, the algorithms or methods they implement, and the stage of development at which they are. Based on a search and extraction of results methodology, this work was able to effectively obtain the current state of the research area. In addition, the exhaustive study of the studies analyzed allowed for a discussion to be established concerning the main difficulties, as well as the gaps and open problems detected for the SPS. The results shown in this study may provide a base for future research on the subject.","title":"Survey of Smart Parking Systems"},"admin":{"seal":true},"id":"9cf511bab39445ba9745feb43d7493dd","created_date":"2020-06-03T00:02:28Z"} diff --git a/python/tests/files/example_fileset_file_ingest_result.json b/python/tests/files/example_fileset_file_ingest_result.json new file mode 100644 index 00000000..94ae4cc6 --- /dev/null +++ b/python/tests/files/example_fileset_file_ingest_result.json @@ -0,0 +1,20 @@ +{"archiveorg_item_name":"springernature.figshare.com-7767695-v1","cdx":{"datetime":"20200223225704","mimetype":"text/html","sha1b32":"37T7GPFQ27ZL6X56G47QKDBZNO5GKTYC","sha1hex":"dfe7f33cb0d7f2bf5fbe373f050c396bba654f02","status_code":200,"surt":"com,figshare,springernature)/articles/landsat_ndvi_difference_1990_2018_tile_6/7767695/1","url":"https://springernature.figshare.com/articles/Landsat_NDVI_difference_1990_2018_Tile_6/7767695/1","warc_csize":6423,"warc_offset":6370648,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200223215122016-26231-26295-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200223225636857-26294-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"0e3019c06b0bc9144d2d9cde84d14ffa","mimetype":"text/html","sha1hex":"dfe7f33cb0d7f2bf5fbe373f050c396bba654f02","sha256hex":"093d7c222bb294b95a0cd55b6285e29d78415460c7b64bb100bcf5e2bcdeb24c","size_bytes":26422},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7767695.v1","https://springernature.figshare.com/articles/Landsat_NDVI_difference_1990_2018_Tile_6/7767695/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"dfc41b617564f99a12e6077a6208876f","mimetype":"image/tiff","path":"NDVI_Diff_1990_2018_T06.tif","platform_url":"https://ndownloader.figshare.com/files/14460875","sha1":"6fb020064da66bb7a666c17555611cf6820fc9ae","sha256":"2febad53ff0f163a18d7cbb913275bf99ed2544730cda191458837e2b0da9d18","size":410631015,"status":"success"}],"platform_base_url":null,"platform_domain":"springernature.figshare.com","platform_id":"7767695","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7767695.v1","ext_ids":{"doi":"10.6084/m9.figshare.7767695.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"bngz3appb5g5vkb4gxjzitmqzu"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7767695.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200223225704","terminal_sha1hex":"dfe7f33cb0d7f2bf5fbe373f050c396bba654f02","terminal_status_code":200,"terminal_url":"https://springernature.figshare.com/articles/Landsat_NDVI_difference_1990_2018_Tile_6/7767695/1"},"total_size":410631015} +{"archiveorg_item_name":"figshare.com-3807984-v1","cdx":{"datetime":"20200213033157","mimetype":"text/html","sha1b32":"X6CTKDY5QDKG376RGYHQ6KUJRL6MJJS6","sha1hex":"bf85350f1d80d46dffd1360f0f2a898afcc4a65e","status_code":200,"surt":"com,figshare)/articles/20150722_qe5_uplc8_rjc_collab_db_4067_01_raw/3807984/1","url":"https://figshare.com/articles/20150722_QE5_UPLC8_RJC_COLLAB_DB_4067_01_raw/3807984/1","warc_csize":6114,"warc_offset":45362828,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200213031757373-10606-10668-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200213033034289-10620-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"4128902a230cd174d43b9fa41d729510","mimetype":"text/html","sha1hex":"bf85350f1d80d46dffd1360f0f2a898afcc4a65e","sha256hex":"fbcf66355bfe726cbf087989459af5e7052c6f0d65e566542ac914e2665f75b6","size_bytes":23637},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.3807984.v1","https://figshare.com/articles/20150722_QE5_UPLC8_RJC_COLLAB_DB_4067_01_raw/3807984/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"03a540a431cd6adf5fd214ba9cdb9a08","mimetype":"application/octet-stream","path":"20150722_QE5_UPLC8_RJC_COLLAB_DB_4067_01.raw","platform_url":"https://ndownloader.figshare.com/files/5930763","sha1":"8454fe535ac60b5f02d71a78f332b98b507de6d2","sha256":"b8967de9381a4f8c8b361b21b09605602b9250799d00f5d43f5be97612f01e18","size":1827479779,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"3807984","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.3807984.v1","ext_ids":{"doi":"10.6084/m9.figshare.3807984.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"b63ugovrtvestd32k3d6kzo5iq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.3807984.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200213033157","terminal_sha1hex":"bf85350f1d80d46dffd1360f0f2a898afcc4a65e","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/20150722_QE5_UPLC8_RJC_COLLAB_DB_4067_01_raw/3807984/1"},"total_size":1827479779} +{"archiveorg_item_name":"figshare.com-873634-v5","cdx":{"datetime":"20200215191649","mimetype":"text/html","sha1b32":"5W4NNJWLAPAJKLB7WTZTAU2RR7IO3MW2","sha1hex":"edb8d6a6cb03c0952c3fb4f33053518fd0edb2da","status_code":200,"surt":"com,figshare)/articles/_research_proposal_mutation_inheritance_the_long_term_effects_of_naturally_occurring_radiation_on_living_systems/873634/5","url":"https://figshare.com/articles/_Research_Proposal_Mutation_Inheritance_The_Long_Term_Effects_of_Naturally_Occurring_Radiation_on_Living_Systems/873634/5","warc_csize":6944,"warc_offset":25128858,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200215190429339-14296-14362-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200215191529091-14307-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"4d334dac22835b23f7fcd4aa6802b393","mimetype":"text/html","sha1hex":"edb8d6a6cb03c0952c3fb4f33053518fd0edb2da","sha256hex":"45a0fd7db2c88199bbe666a418c383036e72c90f2481b694e3ce766fa487e83f","size_bytes":34155},"fileset_file":{"cdx":{"datetime":"20200212204410","mimetype":"binary/octet-stream","sha1b32":"4I4HQS45AZEKSHDPJPAQSNTQG5GA2EP6","sha1hex":"e238784b9d0648a91c6f4bc1093670374c0d11fe","status_code":200,"surt":"com,amazonaws,s3-eu-west-1)/pfigshare-u-files/1798358/mutationinheritance.pdf","url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1798358/MutationInheritance.pdf","warc_csize":139934,"warc_offset":17465253,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200212203938614-09941-10005-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200212204347568-09946-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_meta":[{"md5hex":"57f8cb4fe96e7c55caefcb40e2d5e74f","mimetype":"application/pdf","sha1hex":"e238784b9d0648a91c6f4bc1093670374c0d11fe","sha256hex":"7e403da8055c05bef83051956f523f7e98763732bc93904743f6cdf1865de04d","size_bytes":149128}],"terminal":{"terminal_dt":"20200212204410","terminal_status_code":200,"terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1798358/MutationInheritance.pdf"}},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.873634.v5","https://figshare.com/articles/_Research_Proposal_Mutation_Inheritance_The_Long_Term_Effects_of_Naturally_Occurring_Radiation_on_Living_Systems/873634/5"],"html_biblio":{"abstract":"This research proposal seeks to investigate the possibility that naturally occurring radiation (background radiation) could alter species composition (genotype → phenotype) over extended timeframes. We look to establish a parallel between changed levels of natural radiation and significant evolutionary events. In particular, we will draw our attention to the establishment of a modern atmosphere and the changes in radiation levels that came thereafter. Our study will focus less on early microbial life, and more on modern vertebrates/invertebrates and the subsequent colonization of land species that took place after the ozone layer was established. By implementing a reliable framework in which to test and evaluate genetic variance over successive generations, we will demonstrate that naturally occurring radiation is more influential in creating genetic variance than previously thought. In doing so, we hope to create a standard model for determining expectancies in population genetics based on exposure to multiple levels of radiation.","container_name":"figshare","contrib_names":["Rodriguez, Tommy"],"doi":"10.6084/m9.figshare.873634.v5","pdf_fulltext_url":"https://figshare.com/articles/_Research_Proposal_Mutation_Inheritance_The_Long_Term_Effects_of_Naturally_Occurring_Radiation_on_Living_Systems/873634/files/1798358.pdf","publisher":"figshare","release_date":"2014-11-18","title":"[Research Proposal] Mutation & Inheritance: The Long Term Effects of Naturally Occurring Radiation on Living Systems"},"ingest_strategy":"web-file","manifest":[{"md5":"57f8cb4fe96e7c55caefcb40e2d5e74f","mimetype":"application/pdf","path":"Mutation&Inheritance.pdf","platform_url":"https://ndownloader.figshare.com/files/1798358","sha1":"57f8cb4fe96e7c55caefcb40e2d5e74f","sha256":"7e403da8055c05bef83051956f523f7e98763732bc93904743f6cdf1865de04d","size":149128,"status":"success","terminal_dt":"20200212204410","terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1798358/MutationInheritance.pdf"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"873634","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.873634.v5","ext_ids":{"doi":"10.6084/m9.figshare.873634.v5"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"b62yygefaneexcklxfti2lmwoi"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.873634.v5","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200215191649","terminal_sha1hex":"edb8d6a6cb03c0952c3fb4f33053518fd0edb2da","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/_Research_Proposal_Mutation_Inheritance_The_Long_Term_Effects_of_Naturally_Occurring_Radiation_on_Living_Systems/873634/5"},"total_size":149128} +{"archiveorg_item_name":"figshare.com-4609288-v1","cdx":{"datetime":"20200210051725","mimetype":"text/html","sha1b32":"5NYNWPBAWI3BSBXJTJ63YJPBSW6KMXR5","sha1hex":"eb70db3c20b2361906e99a7dbc25e195bca65e3d","status_code":200,"surt":"com,figshare)/articles/digestion_7_from_exp_it13_leica_register/4609288/1","url":"https://figshare.com/articles/DIgestion_7_from_EXP_IT13_LEICA_register/4609288/1","warc_csize":6029,"warc_offset":37698022,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200210050626489-05335-05397-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200210051705009-05349-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"1e6889267abd940e3ce2ef71bc7d4f4f","mimetype":"text/html","sha1hex":"eb70db3c20b2361906e99a7dbc25e195bca65e3d","sha256hex":"f74bd9c54030b717e0ca79b2670410ef771323a487fdd887cf41f5db4f3c7b41","size_bytes":22742},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.4609288.v1","https://figshare.com/articles/DIgestion_7_from_EXP_IT13_LEICA_register/4609288/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"86e1cbe9220739e6dcbbb5ad832df3a4","mimetype":"application/octet-stream","path":"150816 dig7.lif","platform_url":"https://ndownloader.figshare.com/files/7495531","sha1":"e18476865fbf40c92366122b56cb1228701c5519","sha256":"d62db427ac1bcb0ca09814a8b665d1c2b09ad1f34a03bf4e589eab6ae316d712","size":277695787,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"4609288","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.4609288.v1","ext_ids":{"doi":"10.6084/m9.figshare.4609288.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"duewk6zaurgxvj2hczupf7zuha"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.4609288.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200210051725","terminal_sha1hex":"eb70db3c20b2361906e99a7dbc25e195bca65e3d","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/DIgestion_7_from_EXP_IT13_LEICA_register/4609288/1"},"total_size":277695787} +{"archiveorg_item_name":"figshare.com-1425046-v2","cdx":{"datetime":"20200220172238","mimetype":"text/html","sha1b32":"XNELHLDX7EWMB5JA7RLWRIFVYXPDYS5G","sha1hex":"bb48b3ac77f92cc0f520fc5768a0b5c5de3c4ba6","status_code":200,"surt":"com,figshare)/articles/s14_1/1425046/2","url":"https://figshare.com/articles/S14_1/1425046/2","warc_csize":6182,"warc_offset":119019093,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200220164114618-21546-21609-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200220172053663-21589-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"d9419376cf92a582eaa622a22e7ab7a2","mimetype":"text/html","sha1hex":"bb48b3ac77f92cc0f520fc5768a0b5c5de3c4ba6","sha256hex":"c41ee64c91bf32eecd9f4ab317e3f2e1499b6bf8ccd3b100dfa3433896d1eb00","size_bytes":24693},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.1425046.v2","https://figshare.com/articles/S14_1/1425046/2"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"5ad9b122f0e03a458a2ca2b94f88c279","mimetype":"application/zip","path":"S14_1.zip","platform_url":"https://ndownloader.figshare.com/files/2081115","sha1":"0b31d3e7d1e8423195f25f07bc30fcdbbef7eabe","sha256":"cf745f2b7c22bed50706dc57d553bb80e6a5339252bdd0033a6fdef44dfdc853","size":193762539,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"1425046","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.1425046.v2","ext_ids":{"doi":"10.6084/m9.figshare.1425046.v2"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"asbhbykb7bbuzjwmdzp65tlnwu"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.1425046.v2","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200220172238","terminal_sha1hex":"bb48b3ac77f92cc0f520fc5768a0b5c5de3c4ba6","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/S14_1/1425046/2"},"total_size":193762539} +{"archiveorg_item_name":"figshare.com-155705-v1","cdx":{"datetime":"20200302100511","mimetype":"text/html","sha1b32":"FOQVA5RJVIRKE27DCV3VMNQXANNNEVVC","sha1hex":"2ba1507629aa22a26be31577563617035ad256a2","status_code":200,"surt":"com,figshare)/articles/genomic_signal_processing_one_scientist_s_quest/155705/1","url":"https://figshare.com/articles/Genomic_Signal_Processing_one_scientist_s_quest/155705/1","warc_csize":6768,"warc_offset":92251214,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200302092906472-35659-35722-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200302100304468-35694-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"838a7da2e12f06b8dac16865e0c9375f","mimetype":"text/html","sha1hex":"2ba1507629aa22a26be31577563617035ad256a2","sha256hex":"d4112a7f0ff3f3cd6113678008c8505d8d7d8ac11ecf014c49f08f6a93f6f27f","size_bytes":30273},"fileset_file":{"cdx":{"datetime":"20200214191603","mimetype":"binary/octet-stream","sha1b32":"WX25SVMOOQYM7QT73YVFQ7HEIFBYF3KR","sha1hex":"b5f5d9558e7430cfc27fde2a587ce4414382ed51","status_code":200,"surt":"com,amazonaws,s3-eu-west-1)/pfigshare-u-files/482151/gsponescientistsquest.pdf","url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/482151/GSPonescientistsquest.pdf","warc_csize":305643,"warc_offset":144050325,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200214182939473-13190-13253-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200214191434645-13242-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_meta":[{"md5hex":"b7cf070a3a21fbf3d96abf31c7331658","mimetype":"application/pdf","sha1hex":"b5f5d9558e7430cfc27fde2a587ce4414382ed51","sha256hex":"a356ae5fca5b8c105ce91f14719a5ec5d40bff31cea59fcf14fd8e0717c45db7","size_bytes":323355}],"terminal":{"terminal_dt":"20200214191603","terminal_status_code":200,"terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/482151/GSPonescientistsquest.pdf"}},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.155705.v1","https://figshare.com/articles/Genomic_Signal_Processing_one_scientist_s_quest/155705/1"],"ingest_strategy":"web-file","manifest":[{"md5":"b7cf070a3a21fbf3d96abf31c7331658","mimetype":"application/pdf","path":"GSP-one-scientists-quest.pdf","platform_url":"https://ndownloader.figshare.com/files/482151","sha1":"b7cf070a3a21fbf3d96abf31c7331658","sha256":"a356ae5fca5b8c105ce91f14719a5ec5d40bff31cea59fcf14fd8e0717c45db7","size":323355,"status":"success","terminal_dt":"20200214191603","terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/482151/GSPonescientistsquest.pdf"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"155705","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.155705.v1","ext_ids":{"doi":"10.6084/m9.figshare.155705.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"b2qxpak6cjbypcipjlsz6fk2ge"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.155705.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200302100511","terminal_sha1hex":"2ba1507629aa22a26be31577563617035ad256a2","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Genomic_Signal_Processing_one_scientist_s_quest/155705/1"},"total_size":323355} +{"archiveorg_item_name":"figshare.com-7883471-v1","cdx":{"datetime":"20200223130023","mimetype":"text/html","sha1b32":"BRGI7RR5RNRJR2N34H4SW7RMKYJBQOZR","sha1hex":"0c4c8fc63d8b6298e9bbe1f92b7e2c5612183b31","status_code":200,"surt":"com,figshare)/articles/gvcf_hg01113/7883471/1","url":"https://figshare.com/articles/gVCF_HG01113/7883471/1","warc_csize":6156,"warc_offset":135768315,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200223120354621-25701-25767-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200223125651217-25749-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"098f543205e102a924b402d6fd5aa2bd","mimetype":"text/html","sha1hex":"0c4c8fc63d8b6298e9bbe1f92b7e2c5612183b31","sha256hex":"937a34711a98d783b602a6c005a364d4025d7c211ca1cb9c5e1d9b6239169183","size_bytes":23652},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7883471.v1","https://figshare.com/articles/gVCF_HG01113/7883471/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"b4a1733982a44b1eb1f4ee957eb7bf41","mimetype":"application/x-gzip","path":"HG01113.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14675255","sha1":"b907f6e243820a09f10c001900fb99141537ac63","sha256":"ebae318f297acd3ce28f6d7bda92eaf863395a18594908dda016fe2632f7fd38","size":3558964608,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7883471","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7883471.v1","ext_ids":{"doi":"10.6084/m9.figshare.7883471.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"blw4at5lxvhcldwjapqipierkq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7883471.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200223130023","terminal_sha1hex":"0c4c8fc63d8b6298e9bbe1f92b7e2c5612183b31","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_HG01113/7883471/1"},"total_size":3558964608} +{"archiveorg_item_name":"figshare.com-1328466-v1","cdx":{"datetime":"20200213025244","mimetype":"text/html","sha1b32":"ECWCYMZ2PEZG7ZU2PVVNDTRXDR5BIFCM","sha1hex":"20ac2c333a79326fe69a7d6ad1ce371c7a14144c","status_code":200,"surt":"com,figshare)/articles/pecten_maximus_est_database/1328466/1","url":"https://figshare.com/articles/Pecten_maximus_EST_database/1328466/1","warc_csize":6335,"warc_offset":131119250,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200213020718819-10319-10382-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200213025003763-10366-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"a7d7b5e7e48d883ec67b921b4362bc82","mimetype":"text/html","sha1hex":"20ac2c333a79326fe69a7d6ad1ce371c7a14144c","sha256hex":"60f8da0818a1890aa3a9d01da3c9d36dc4934d9ef8a7df91b492e7a673dd055c","size_bytes":24908},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.1328466.v1","https://figshare.com/articles/Pecten_maximus_EST_database/1328466/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"42204274a103103ef8872ecf5cfb2bdd","mimetype":"text/plain","path":"pecten_maximus.fasta","platform_url":"https://ndownloader.figshare.com/files/1939770","sha1":"b0ac5f991fad6168deceff8cb144dd42f32ed8a7","sha256":"e81abab77b76ad7a4e6271777071391793623d17372fcd3477efd3fa0dd2d541","size":165410129,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"1328466","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.1328466.v1","ext_ids":{"doi":"10.6084/m9.figshare.1328466.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"azywvawlkfdt3g3tewt4zahfca"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.1328466.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200213025244","terminal_sha1hex":"20ac2c333a79326fe69a7d6ad1ce371c7a14144c","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Pecten_maximus_EST_database/1328466/1"},"total_size":165410129} +{"archiveorg_item_name":"figshare.com-7891043-v1","cdx":{"datetime":"20200210043948","mimetype":"text/html","sha1b32":"IRLPGL5NVEK74PJ2KOEYBXPVYUWBCYTW","sha1hex":"4456f32fada915fe3d3a538980ddf5c52c116276","status_code":200,"surt":"com,figshare)/articles/gvcf_na18907/7891043/1","url":"https://figshare.com/articles/gVCF_NA18907/7891043/1","warc_csize":6157,"warc_offset":17501773,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200210041721185-05270-05334-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200210043919713-05302-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"60c2be90859130938e3dae732b511047","mimetype":"text/html","sha1hex":"4456f32fada915fe3d3a538980ddf5c52c116276","sha256hex":"cdc0f95c1127389881aae42178f018b2fd373b7c46e34c7656d2085d17c3cad7","size_bytes":23652},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7891043.v1","https://figshare.com/articles/gVCF_NA18907/7891043/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"8b7bb4984f8cb05a971df28b1188601f","mimetype":"application/x-gzip","path":"NA18907.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14831225","sha1":"3abe9b16a7dd03f1872b1f7e60230cc2472f5363","sha256":"b8520f5b51fd3c8b9540a2fae43546441a860548fe696e05ca94bde4e4d7e55a","size":5080602128,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7891043","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7891043.v1","ext_ids":{"doi":"10.6084/m9.figshare.7891043.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"doq6plp2pfbbjaekby6qpboclq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7891043.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200210043948","terminal_sha1hex":"4456f32fada915fe3d3a538980ddf5c52c116276","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_NA18907/7891043/1"},"total_size":5080602128} +{"archiveorg_item_name":"figshare.com-7944293-v1","cdx":{"datetime":"20200219161852","mimetype":"text/html","sha1b32":"BPOM2XH7LDGXBZXHOFI7R2X5G6DV6JSF","sha1hex":"0bdccd5cff58cd70e6e77151f8eafd37875f2645","status_code":200,"surt":"com,figshare)/articles/gvcf_na18504/7944293/1","url":"https://figshare.com/articles/gVCF_NA18504/7944293/1","warc_csize":6161,"warc_offset":47228503,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200219151726124-20609-20673-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200219161738665-20670-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"388ca467f21de95c62fab428ff1f9594","mimetype":"text/html","sha1hex":"0bdccd5cff58cd70e6e77151f8eafd37875f2645","sha256hex":"db5c1adaca1b0466b3eac7d15b9ea348360e0d31e3be062f2301b8563428a740","size_bytes":23648},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7944293.v1","https://figshare.com/articles/gVCF_NA18504/7944293/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"52473c022862137bd2f0262ff5654763","mimetype":"application/x-gzip","path":"NA18504.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14789654","sha1":"b0a9dfe52fba77755eef81ea5ca1224218495975","sha256":"0931bb9376f0a95494b9f78e1bfb195e75f931b5ed8c53253ec6eb9acbea4165","size":9241364045,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7944293","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7944293.v1","ext_ids":{"doi":"10.6084/m9.figshare.7944293.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"dokkkeowufarnc7hc2lictjfpa"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7944293.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200219161852","terminal_sha1hex":"0bdccd5cff58cd70e6e77151f8eafd37875f2645","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_NA18504/7944293/1"},"total_size":9241364045} +{"archiveorg_item_name":"figshare.com-96798-v1","cdx":{"datetime":"20200216095334","mimetype":"text/html","sha1b32":"EZJKDVBIPTSAZB43VHRKL6IXQ2UBLJRC","sha1hex":"2652a1d4287ce40c879ba9e2a5f91786a815a622","status_code":200,"surt":"com,figshare)/articles/fuelwood,_savannas,_and_climate_change:_integrating_modeling,_field_experimentation,_and_optical_and_radar_remote_sensing/96798/1","url":"https://figshare.com/articles/Fuelwood,_Savannas,_and_Climate_Change:_Integrating_Modeling,_Field_Experimentation,_and_Optical_and_Radar_Remote_Sensing/96798/1","warc_csize":6488,"warc_offset":121408362,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200216090256643-15150-15213-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200216095021545-15201-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"9733a4d39f63227597b185510119c53d","mimetype":"text/html","sha1hex":"2652a1d4287ce40c879ba9e2a5f91786a815a622","sha256hex":"6352c8db9c37c4ff806d673a0adc204e70e904156b8caae61065bc4db2dc852f","size_bytes":27330},"fileset_file":{"cdx":{"datetime":"20200221054157","mimetype":"binary/octet-stream","sha1b32":"A6NUHHMLZR4ADDLH7WBXXDHDBVVO2POU","sha1hex":"079b439d8bcc78018d67fd837b8ce30d6aed3dd4","status_code":200,"surt":"com,amazonaws,s3-eu-west-1)/pfigshare-u-files/101362/final_nessf11_tredennick.pdf","url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/101362/FINAL_NESSF11_Tredennick.pdf","warc_csize":2831911,"warc_offset":148846417,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200221042341296-22665-22731-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200221053650453-22721-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_meta":[{"md5hex":"955ed359be13ad13d4ab39cc6e0d1795","mimetype":"application/pdf","sha1hex":"079b439d8bcc78018d67fd837b8ce30d6aed3dd4","sha256hex":"6660a07237da9050e1ed168b491f87057af5cd33e3cbe934ea2cb26f8c125d64","size_bytes":3141366}],"terminal":{"terminal_dt":"20200221054157","terminal_status_code":200,"terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/101362/FINAL_NESSF11_Tredennick.pdf"}},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.96798.v1","https://figshare.com/articles/Fuelwood,_Savannas,_and_Climate_Change:_Integrating_Modeling,_Field_Experimentation,_and_Optical_and_Radar_Remote_Sensing/96798/1"],"ingest_strategy":"web-file","manifest":[{"md5":"955ed359be13ad13d4ab39cc6e0d1795","mimetype":"application/pdf","path":"FINAL_NESSF11_Tredennick.pdf","platform_url":"https://ndownloader.figshare.com/files/101362","sha1":"955ed359be13ad13d4ab39cc6e0d1795","sha256":"6660a07237da9050e1ed168b491f87057af5cd33e3cbe934ea2cb26f8c125d64","size":3141366,"status":"success","terminal_dt":"20200221054157","terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/101362/FINAL_NESSF11_Tredennick.pdf"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"96798","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.96798.v1","ext_ids":{"doi":"10.6084/m9.figshare.96798.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"do72fzxjlzhpjjub7pscjymvde"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.96798.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200216095334","terminal_sha1hex":"2652a1d4287ce40c879ba9e2a5f91786a815a622","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Fuelwood,_Savannas,_and_Climate_Change:_Integrating_Modeling,_Field_Experimentation,_and_Optical_and_Radar_Remote_Sensing/96798/1"},"total_size":3141366} +{"archiveorg_item_name":"figshare.com-7059653-v1","cdx":{"datetime":"20200219071230","mimetype":"text/html","sha1b32":"PXELMU4HEPJASJLGBIPTWWJ7R3LVTMAA","sha1hex":"7dc8b6538723d20925660a1f3b593f8ed759b000","status_code":200,"surt":"com,figshare)/articles/gwr_air_temperature_for_year_2012/7059653/1","url":"https://figshare.com/articles/GWR_air_temperature_for_year_2012/7059653/1","warc_csize":6373,"warc_offset":82312661,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200219062845359-19514-19579-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200219071046185-19559-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"030ab36d3053dcaa7ad14afb9eec9ab9","mimetype":"text/html","sha1hex":"7dc8b6538723d20925660a1f3b593f8ed759b000","sha256hex":"760244a3cdca50ee50a41b3285ad7f3dff6045eed4f72e61f8387acaf9c1c20d","size_bytes":25691},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7059653.v1","https://figshare.com/articles/GWR_air_temperature_for_year_2012/7059653/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"91ae25cbb68a92d067da63c70855e979","mimetype":"application/x-hdf","path":"Tair_GWR_2012.nc","platform_url":"https://ndownloader.figshare.com/files/12983516","sha1":"b0bb6b49c276bea61e90dfb553013bc7b0ae6e0f","sha256":"c7b9aba099374b613e636d0c84f321b1301834f77ba346f9b7aa8d0626134b55","size":168632217,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7059653","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7059653.v1","ext_ids":{"doi":"10.6084/m9.figshare.7059653.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"aq5xmvnwz5anhmumsvlnc3b7qi"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7059653.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200219071230","terminal_sha1hex":"7dc8b6538723d20925660a1f3b593f8ed759b000","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/GWR_air_temperature_for_year_2012/7059653/1"},"total_size":168632217} +{"archiveorg_item_name":"figshare.com-1290730-v1","cdx":{"datetime":"20200218045018","mimetype":"text/html","sha1b32":"DIGYAYMD2GXMU4SKQIO4QGJFBIR37VVA","sha1hex":"1a0d806183d1aeca724a821dc819250a23bfd6a0","status_code":200,"surt":"com,figshare)/articles/difficult_airway_in_thyroid_surgery/1290730/1","url":"https://figshare.com/articles/Difficult_airway_in_thyroid_surgery/1290730/1","warc_csize":6943,"warc_offset":67583985,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200218040911124-17891-17955-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200218044808327-17934-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"24b87928eb315579ce5263884b4bcbec","mimetype":"text/html","sha1hex":"1a0d806183d1aeca724a821dc819250a23bfd6a0","sha256hex":"4fbafb454edccba5ab0ebaf90e40f6709ce99d9326ab9a18e2325a02f558c1be","size_bytes":32119},"fileset_file":{"cdx":{"datetime":"20200218045039","mimetype":"binary/octet-stream","sha1b32":"RS5T57Z2H6ZIMYM35YSPSK6QYJXJ5EEF","sha1hex":"8cbb3eff3a3fb286619bee24f92bd0c26e9e9085","status_code":200,"surt":"com,amazonaws,s3-eu-west-1)/pfigshare-u-files/1867824/revistaorl201118_viaaereatiroides.pdf","url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1867824/revistaorl201118_viaaereatiroides.pdf","warc_csize":356344,"warc_offset":57825647,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200218040911124-17891-17955-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200218044857403-17936-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_meta":[{"md5hex":"bdc2ca5f3b996c24890ee706a5d91fce","mimetype":"application/pdf","sha1hex":"8cbb3eff3a3fb286619bee24f92bd0c26e9e9085","sha256hex":"9acc5aac353c6ff40fb1792d5a41b2f4306bfb75f016965100b94a07de24c5e7","size_bytes":410344}],"terminal":{"terminal_dt":"20200218045039","terminal_status_code":200,"terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1867824/revistaorl201118_viaaereatiroides.pdf"}},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.1290730.v1","https://figshare.com/articles/Difficult_airway_in_thyroid_surgery/1290730/1"],"html_biblio":{"abstract":"Background: Thyroid surgery has been considered as a risk factor for the management of difficult airway. Method: We review the relevant literature related to the difficult airway and thyroid surgery. Comments: Endothoracic goiter, deformity of the airway, compression of the airway and thyroid cancer do not appear to have an influence on the difficulty for intubation. Classical parameters have a better predictive value: mouth opening less than 35 mm, Mallampati II-IV, short neck, neck movility less than 80 degrees, thyromental distance less than 60 mm, retrognathia, Cormack index III-IV, macroglossia, inability to protude the teeth and body mass index (BMI) greater than 35 Kg/m2. Conclusions: According to the literature, thyroid disease does not increase the risk for a difficult airway (ventilation / intubation). The risk of an unexpected difficult airway (false negative) cannot be eliminated. All patients must undergo an assessment of the airway","container_name":"figshare","contrib_names":["Javier CuelloAzcárate, Jesús","Pardal-Refoyo, Jose Luis","Almudena Romero-Rapado, ","Pilar Toro-Pinilla, María"],"doi":"10.6084/m9.figshare.1290730.v1","pdf_fulltext_url":"https://figshare.com/articles/Difficult_airway_in_thyroid_surgery/1290730/files/1867824.pdf","publisher":"figshare","release_date":"2015-01-18","title":"Difficult airway in thyroid surgery"},"ingest_strategy":"web-file","manifest":[{"md5":"bdc2ca5f3b996c24890ee706a5d91fce","mimetype":"application/pdf","path":"revistaorl201118_viaaereatiroides.pdf","platform_url":"https://ndownloader.figshare.com/files/1867824","sha1":"bdc2ca5f3b996c24890ee706a5d91fce","sha256":"9acc5aac353c6ff40fb1792d5a41b2f4306bfb75f016965100b94a07de24c5e7","size":410344,"status":"success","terminal_dt":"20200218045039","terminal_url":"https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/1867824/revistaorl201118_viaaereatiroides.pdf"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"1290730","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.1290730.v1","ext_ids":{"doi":"10.6084/m9.figshare.1290730.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"eftssxll5ngipgmue5ahnsan3i"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.1290730.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200218045018","terminal_sha1hex":"1a0d806183d1aeca724a821dc819250a23bfd6a0","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Difficult_airway_in_thyroid_surgery/1290730/1"},"total_size":410344} +{"archiveorg_item_name":"figshare.com-7879991-v1","cdx":{"datetime":"20200301215510","mimetype":"text/html","sha1b32":"IINKFPDJIAXYVLDYVUXSO6EGJPNEKR6P","sha1hex":"421aa2bc69402f8aac78ad2f2778864bda4547cf","status_code":200,"surt":"com,figshare)/articles/gvcf_hg00530/7879991/1","url":"https://figshare.com/articles/gVCF_HG00530/7879991/1","warc_csize":6343,"warc_offset":45356,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200301213356805-35009-35074-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200301215511721-35022-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"2a01939b84dabdaa28ceb363d0319442","mimetype":"text/html","sha1hex":"421aa2bc69402f8aac78ad2f2778864bda4547cf","sha256hex":"7b5f2f51bad52c73d7325d991fc77b7786957b1a9b356fb5bf58f5efa402a9a4","size_bytes":26774},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7879991.v1","https://figshare.com/articles/gVCF_HG00530/7879991/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"86c5baf03e03173d200e2e1a1cc6fd79","mimetype":"application/x-gzip","path":"HG00530.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14670497","sha1":"c3f0cc4b4d65b730d7b4cafcfc621d187f301ba0","sha256":"124f0fd047046e892275d862643e9814c8db63ef5cff8c7d44c25ec8c8d73841","size":5957749616,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7879991","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7879991.v1","ext_ids":{"doi":"10.6084/m9.figshare.7879991.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"ccata4ww7nekflu2l5rhtn3b3i"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7879991.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200301215510","terminal_sha1hex":"421aa2bc69402f8aac78ad2f2778864bda4547cf","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_HG00530/7879991/1"},"total_size":5957749616} +{"archiveorg_item_name":"figshare.com-4007418-v2","cdx":{"datetime":"20200212070037","mimetype":"text/html","sha1b32":"CFQUF74PASKLKUQYTY5MCOP3THNV4J2S","sha1hex":"116142ff8f0494b552189e3ac139fb99db5e2752","status_code":200,"surt":"com,figshare)/articles/ch4_pasr_data_bin/4007418/2","url":"https://figshare.com/articles/ch4_pasr_data_bin/4007418/2","warc_csize":7001,"warc_offset":116323768,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200212065257603-08978-09042-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200212065830590-08986-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"bdab1e11a9e10714992091fa4db91935","mimetype":"text/html","sha1hex":"116142ff8f0494b552189e3ac139fb99db5e2752","sha256hex":"77562652fa99ec2223b3daec53818e2024e98c191207595bccc9f35f96da0e0a","size_bytes":31495},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.4007418.v2","https://figshare.com/articles/ch4_pasr_data_bin/4007418/2"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"7a4bdb4b9329e2937db5b5639ebee7c4","mimetype":"application/zip","path":"ch4_pasr_data.npz","platform_url":"https://ndownloader.figshare.com/files/6463836","sha1":"7e496df234c2d4aa9adbadac99a2009787cd09ef","sha256":"3cf46e8198343fc25c50c1f047c69ac0672cd333f97d355726ac2f6d65e24524","size":202004950,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"4007418","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.4007418.v2","ext_ids":{"doi":"10.6084/m9.figshare.4007418.v2"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"bub6fa6iljfbfaguuxu36etywq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.4007418.v2","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200212070037","terminal_sha1hex":"116142ff8f0494b552189e3ac139fb99db5e2752","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/ch4_pasr_data_bin/4007418/2"},"total_size":202004950} +{"archiveorg_item_name":"figshare.com-8280893-v1","cdx":{"datetime":"20200228033059","mimetype":"text/html","sha1b32":"JCV325BG2DKHSLOXR4VKRVYUJX3LN6ML","sha1hex":"48abbd7426d0d4792dd78f2aa8d7144df6b6f98b","status_code":200,"surt":"com,figshare)/articles/dataset_during_rhythmic_pointing_joint-angles_are_temporary_linked_in_different_synergies_with_similar_relative_joint-angle_coupling_across_task_constraints_/8280893/1","url":"https://figshare.com/articles/Dataset_During_rhythmic_pointing_joint-angles_are_temporary_linked_in_different_synergies_with_similar_relative_joint-angle_coupling_across_task_constraints_/8280893/1","warc_csize":6670,"warc_offset":84669003,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200228023255589-30612-30678-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200228032853298-30662-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"038f0c60fd6b79dfc3ef26d89a74f68f","mimetype":"text/html","sha1hex":"48abbd7426d0d4792dd78f2aa8d7144df6b6f98b","sha256hex":"a7dba4ca5e575b0bdfe2f9d944b157c4b057c5257d2062a67e007da4d9ffa70e","size_bytes":31199},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.8280893.v1","https://figshare.com/articles/Dataset_During_rhythmic_pointing_joint-angles_are_temporary_linked_in_different_synergies_with_similar_relative_joint-angle_coupling_across_task_constraints_/8280893/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"12c3ff84915fc677fef4126cbcf3f52d","mimetype":"application/x-rar","path":"Data Valk et al 2019 SciReports.rar","platform_url":"https://ndownloader.figshare.com/files/15482963","sha1":"fb1d8152c7dd6048f652e1890a6bff6a679e0cea","sha256":"69fc117e07121e2a43320b6a24cf04ffc41d28069a142ae8b44a79b2b52f2320","size":1320490575,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"8280893","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.8280893.v1","ext_ids":{"doi":"10.6084/m9.figshare.8280893.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"db6v4erinjcllo6mq5wmxlvqby"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.8280893.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200228033059","terminal_sha1hex":"48abbd7426d0d4792dd78f2aa8d7144df6b6f98b","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Dataset_During_rhythmic_pointing_joint-angles_are_temporary_linked_in_different_synergies_with_similar_relative_joint-angle_coupling_across_task_constraints_/8280893/1"},"total_size":1320490575} +{"archiveorg_item_name":"figshare.com-7885169-v1","cdx":{"datetime":"20200220015119","mimetype":"text/html","sha1b32":"7IBKPBR3AO5XRAAAW77NGEOUUMS3BUP3","sha1hex":"fa02a7863b03bb788000b7fed311d4a325b0d1fb","status_code":200,"surt":"com,figshare)/articles/gvcf_hg01256/7885169/1","url":"https://figshare.com/articles/gVCF_HG01256/7885169/1","warc_csize":6154,"warc_offset":43108844,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200220014610521-21205-21270-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200220015026708-21210-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"fb96082852b6c510739ca4e3dfc04d4a","mimetype":"text/html","sha1hex":"fa02a7863b03bb788000b7fed311d4a325b0d1fb","sha256hex":"69db668dda74567b7868e74d35695cccf6bc61a43b205cca7014f7b5829ef806","size_bytes":23648},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7885169.v1","https://figshare.com/articles/gVCF_HG01256/7885169/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"b4ae15f84e5bb15a1856df21071f601b","mimetype":"application/x-gzip","path":"HG01256.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14677301","sha1":"2d6b4b16baf60eaf5c64f6ea9fb8aec8ab8c8e86","sha256":"1d7c859b2cf0dce0536520d434992e180002d193fd7b650a1d501c4fe491b990","size":5896179103,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7885169","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7885169.v1","ext_ids":{"doi":"10.6084/m9.figshare.7885169.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"dlsawja46fahtn7llhrhaeh6gu"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7885169.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200220015119","terminal_sha1hex":"fa02a7863b03bb788000b7fed311d4a325b0d1fb","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_HG01256/7885169/1"},"total_size":5896179103} +{"archiveorg_item_name":"figshare.com-1504026-v1","cdx":{"datetime":"20200302200059","mimetype":"text/html","sha1b32":"DUX3N7QY5EQNYO3G6P3AETT7J3FMGECQ","sha1hex":"1d2fb6fe18e920dc3b66f3f6024e7f4ecac31050","status_code":200,"surt":"com,figshare)/articles/fish_proj800_trans_part3/1504026/1","url":"https://figshare.com/articles/FISH_PROJ800_TRANS_part3/1504026/1","warc_csize":7072,"warc_offset":788146,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200302190810451-34977-35042-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200302200055284-35013-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"679e6dc9a4b89ee8012a994bc53f53d1","mimetype":"text/html","sha1hex":"1d2fb6fe18e920dc3b66f3f6024e7f4ecac31050","sha256hex":"432fbd760f0e56f9185fb83f41acdb92c15fbaaa14c633e50f70ded61d55cf4a","size_bytes":33179},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.1504026.v1","https://figshare.com/articles/FISH_PROJ800_TRANS_part3/1504026/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"ae49f8baacd5ca26fef939db1afe33f3","mimetype":"application/zip","path":"FISH_PROJ800_TRANS_part3.zip","platform_url":"https://ndownloader.figshare.com/files/3378803","sha1":"9fb06893afb498be679c959f77f6f0f618247d54","sha256":"2bb53ab934ba4de9706ee1f9dc08fd30ccd3746e41b39aedf716cc1d5dc33e49","size":200446950,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"1504026","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.1504026.v1","ext_ids":{"doi":"10.6084/m9.figshare.1504026.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"c65bw4qtcjgoto7kqgklyp5fve"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.1504026.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200302200059","terminal_sha1hex":"1d2fb6fe18e920dc3b66f3f6024e7f4ecac31050","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/FISH_PROJ800_TRANS_part3/1504026/1"},"total_size":200446950} +{"archiveorg_item_name":"figshare.com-7895180-v1","cdx":{"datetime":"20200224231540","mimetype":"text/html","sha1b32":"WNSGMXOO3NNYUBLRFOG33B7GJYXRBKPT","sha1hex":"b364665dcedb5b8a05712b8dbd87e64e2f10a9f3","status_code":200,"surt":"com,figshare)/articles/gvcf_hg03190/7895180/1","url":"https://figshare.com/articles/gVCF_HG03190/7895180/1","warc_csize":6348,"warc_offset":146971019,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200224223743553-26853-26919-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200224231109634-26881-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"c6912cf73aa77856598d7ec0391235d4","mimetype":"text/html","sha1hex":"b364665dcedb5b8a05712b8dbd87e64e2f10a9f3","sha256hex":"9060112d112f26bff392fd895c37629b0b30b859a33ab80c98bf195d63ebed47","size_bytes":26766},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7895180.v1","https://figshare.com/articles/gVCF_HG03190/7895180/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"36307bca6fc019e5f0faff1d5264c3c1","mimetype":"application/x-gzip","path":"HG03190.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14702432","sha1":"9a259e2741612b1b0611dc5b4b6400ca6f74b063","sha256":"707c614fdfd274f64d4b7e82596a157f0b3faecf1c6157aabd0a4cc61bc6af23","size":5148586592,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7895180","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7895180.v1","ext_ids":{"doi":"10.6084/m9.figshare.7895180.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"cfmk2v7sdjfurerhfj7zooij6u"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7895180.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200224231540","terminal_sha1hex":"b364665dcedb5b8a05712b8dbd87e64e2f10a9f3","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_HG03190/7895180/1"},"total_size":5148586592} +{"archiveorg_item_name":"figshare.com-7932278-v1","cdx":{"datetime":"20200208045817","mimetype":"text/html","sha1b32":"TK2EV6VFMRYO4DXSTQYGJHAWYV3AAPWS","sha1hex":"9ab44afaa56470ee0ef29c30649c16c576003ed2","status_code":200,"surt":"com,figshare)/articles/gvcf_na19795/7932278/1","url":"https://figshare.com/articles/gVCF_NA19795/7932278/1","warc_csize":6158,"warc_offset":30962659,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200208045034449-01427-01489-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200208045800643-01437-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":1,"file_meta":{"md5hex":"79db15c64560af1da31510b50b706c6d","mimetype":"text/html","sha1hex":"9ab44afaa56470ee0ef29c30649c16c576003ed2","sha256hex":"96ff0c36ace08a420d50079ec96be766c56750e8bd1a93604e415aa149d96a57","size_bytes":23648},"fileset_file":{},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7932278.v1","https://figshare.com/articles/gVCF_NA19795/7932278/1"],"ingest_strategy":"archiveorg-file","manifest":[{"md5":"7a9dc016f5b4d369a5a2f2353e57bcb1","mimetype":"application/x-gzip","path":"NA19795.dedup.realigned.recalibrated.hc.g.vcf.gz","platform_url":"https://ndownloader.figshare.com/files/14760185","sha1":"df6ff3a1378a6538aacf8aa06da6d12cd45b94b8","sha256":"3ccb1c811851f5dff59227f2f3deaaa88aeb0c56fcb81eaa3f01a7bdf3546062","size":5574989369,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7932278","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7932278.v1","ext_ids":{"doi":"10.6084/m9.figshare.7932278.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"byoabgtkdvhatpx7sqievxqktq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7932278.v1","release_stage":"published"},"status":"success-file","terminal":{"terminal_dt":"20200208045817","terminal_sha1hex":"9ab44afaa56470ee0ef29c30649c16c576003ed2","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/gVCF_NA19795/7932278/1"},"total_size":5574989369} diff --git a/python/tests/files/example_fileset_ingest_result.json b/python/tests/files/example_fileset_ingest_result.json new file mode 100644 index 00000000..6b460b33 --- /dev/null +++ b/python/tests/files/example_fileset_ingest_result.json @@ -0,0 +1,20 @@ +{"archiveorg_item_name":"zenodo.org-1028059","cdx":{"datetime":"20200211074218","mimetype":"text/html","sha1b32":"N754JJAQWWTHMZMMNECQFOPCSTDWNP7I","sha1hex":"6ffbc4a410b5a676658c690502b9e294c766bfe8","status_code":200,"surt":"org,zenodo)/record/1028059","url":"https://zenodo.org/record/1028059","warc_csize":9720,"warc_offset":79299,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200211071817172-07303-07367-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200211074221315-07339-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"c318f2e8cc85a432e52fbeda2209f5b8","mimetype":"text/html","sha1hex":"6ffbc4a410b5a676658c690502b9e294c766bfe8","sha256hex":"60e27cff98b7b51edae1b3b90cc94930de0309166974bc2b8ee7870c8c2a2e89","size_bytes":40140},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1028059","https://zenodo.org/record/1028059"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Javer, Avelino","Currie, Michael","Hokanson, Jim","Lee, Chee Wai","Li, Kezhi","Yemini, Eviatar","Grundy, Laura J","Li, Chris","Ch'ng, Quee-Lim","Schafer, William R","Kerr, Rex","Brown, André EX"],"doi":"10.123/abc","release_date":"2017-10-20","title":"N2 Schafer Lab N2 (Bristol, UK) | 2010-03-25T10:53:27+00:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"caf4d9fc2c6ebd0d9251ac84e0b6b006","mimetype":"application/x-hdf","path":"N2 on food R_2010_03_25__10_53_27___4___1_features.hdf5","platform_url":"https://zenodo.org/api/files/563203f6-6de5-46d9-b305-ba42604f2508/N2%20on%20food%20R_2010_03_25__10_53_27___4___1_features.hdf5","sha1":"c0669e84e7b9052cc0f342e8ce7d31d59956326a","sha256":"d218694facb90e96a65f008e1205c1514bae54bdb4e1c2296c1c8dcc31dece9d","size":16799750,"status":"success"},{"md5":"961f88355395408ac9464cc1b33dab4a","mimetype":"application/x-hdf","path":"N2 on food R_2010_03_25__10_53_27___4___1.hdf5","platform_url":"https://zenodo.org/api/files/563203f6-6de5-46d9-b305-ba42604f2508/N2%20on%20food%20R_2010_03_25__10_53_27___4___1.hdf5","sha1":"088976277004081586186e81874dc28acc7ef5e7","sha256":"1e4e0025615eac7ebdd779a7d788234386b5d9e72717b2cb782ecf3f49639845","size":218752636,"status":"success"},{"md5":"f0ede89416863c9571eb9081d7dbc2ce","mimetype":"application/zip","path":"N2 on food R_2010_03_25__10_53_27___4___1.wcon.zip","platform_url":"https://zenodo.org/api/files/563203f6-6de5-46d9-b305-ba42604f2508/N2%20on%20food%20R_2010_03_25__10_53_27___4___1.wcon.zip","sha1":"efe9b640003725cb7e5dfd778ec50f242ada4651","sha256":"aedd1d5fd4a8857fa8dda294ec95a2ff473060402aadb4f068fe26b258ad5d71","size":15161284,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1028059","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1028059","ext_ids":{"doi":"10.5281/zenodo.1028059"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaam","work_ident":"5ouiftuywfdzxagyow5e7o4tkm"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1028059","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200211074218","terminal_sha1hex":"6ffbc4a410b5a676658c690502b9e294c766bfe8","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1028059"},"total_size":250713670} +{"archiveorg_item_name":"zenodo.org-1321718","cdx":{"datetime":"20210508144055","mimetype":"text/html","sha1b32":"HOSEK6JDNUX24RWKGGAVQXEASNRHGUS6","sha1hex":"3ba44579236d2fae46ca3181585c80936273525e","status_code":200,"surt":"org,zenodo)/record/1321718","url":"https://zenodo.org/record/1321718","warc_csize":12504,"warc_offset":1159397098,"warc_path":"CC-MAIN-2021-21-1620243988882.7-0021/CC-MAIN-20210508121446-20210508151446-00433.warc.gz"},"file_count":75,"file_meta":{"md5hex":"a7c0a426fc84240e2e248402ee1f62ea","mimetype":"text/html","sha1hex":"3ba44579236d2fae46ca3181585c80936273525e","sha256hex":"6a07f183c60f4d9251961524cbb5e281a8fcd89f02f0c3885e25ebf2ef2ac87d","size_bytes":103838},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1321718","https://zenodo.org/record/1321718"],"html_biblio":{"container_name":"Zenodo","contrib_names":["OpenAIREH2020"],"doi":"10.5281/zenodo.1321718","release_date":"2018-07-31","title":"OpenAIRE LOD Dump"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"5c5a8f8d3f28395c2d2dfb7df607e245","mimetype":"application/gzip","path":"output000001.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000001.nt.gz116447","sha1":"d6651479a5b35592b3543fddb0c39fcf1d1cfb23","sha256":"bf8820d3be4aaffd5a4e9a0d06999b0f66a463e58d2b9f448105d8614dc0a9a7","size":460210138,"status":"success"},{"md5":"47041ed92f823753d279587f99a27040","mimetype":"application/gzip","path":"output000002.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000002.nt.gz116447","sha1":"21dc6edae2bdc32c0e722a9c4f6fc4d543685112","sha256":"132fa756b87c3f146f06cf4de226317ba8e7c4aead18be19e53b64ce95cca544","size":461323860,"status":"success"},{"md5":"4c895fa52f1dc07506dc0fdf6ce51f05","mimetype":"application/gzip","path":"output000003.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000003.nt.gz116447","sha1":"fb331b6955be6a70b9962f85a39f0f4e4a7b1597","sha256":"e20186f58a7bfe9d76db4e20a7bd0dba38ad4dd8103ec3e7cc2b7c9d750f3543","size":464804466,"status":"success"},{"md5":"d20089489d812e97e243740088da5fe4","mimetype":"application/gzip","path":"output000004.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000004.nt.gz116447","sha1":"5d2a95ca7d7903a5371baa831d38cc05850bbeaa","sha256":"765367d95e322e096d066b691b58526db3d4657a5ea50ba056006e20a6cc1c4f","size":470483091,"status":"success"},{"md5":"83d48f92f7347e8d00e58dee6a600123","mimetype":"application/gzip","path":"output000005.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000005.nt.gz116447","sha1":"05894da6b8744b77f7afe802fad8c490812a3e66","sha256":"ba39222501b0a2ac80d72734f7c676469140b0b31bde64746ec0665356ddd8ed","size":467870895,"status":"success"},{"md5":"42e732b78c9cab859f80d6601e88aa9b","mimetype":"application/gzip","path":"output000006.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000006.nt.gz116447","sha1":"aa1bfd1f78efce4e5763ca57acd251b554272053","sha256":"f785324d84112beb3870abe414a9c6f47fb1cca3f24066f0950bd667e9671c71","size":483294294,"status":"success"},{"md5":"731f947294829cb5ebc22a5bc1ba6cbd","mimetype":"application/gzip","path":"output000007.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000007.nt.gz116447","sha1":"f08dd730fa410fb59681c3fe642fc1e011ee2901","sha256":"a5cc22b048de621fe2e0a41bd8b75b707d89a43e7d551f7b43f5e2c059648d97","size":594718851,"status":"success"},{"md5":"cbfe596be3b28d0bc4b523a8c2649677","mimetype":"application/gzip","path":"output000008.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000008.nt.gz116447","sha1":"090c6138431b5a816c11602b18406b39574ec097","sha256":"461043c0fe66bba0ce80f96a0a0fb26785696cd0a35ea6d40b98034d73d2b014","size":572505521,"status":"success"},{"md5":"552a2117f46b61402f4d6193276d384a","mimetype":"application/gzip","path":"output000009.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000009.nt.gz116447","sha1":"b46989626193722f35b38aebdfc1e462da88f7e2","sha256":"a19e9daf497f8add2dba61f6844fc90a8e72c8058068792b63fcc9341d5cc34c","size":565484003,"status":"success"},{"md5":"12d1699f0713f035169c44a72416b036","mimetype":"application/gzip","path":"output000010.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000010.nt.gz116447","sha1":"e709ab15b43e9f77347d379af01fb3d56c8c70df","sha256":"5638979e2c3998128ffc49a3da20b6e4d8410cd1c80d96f5422192977dee74d2","size":557815275,"status":"success"},{"md5":"d0805d8fd67f97c45aaf79b80cf33f38","mimetype":"application/gzip","path":"output000011.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000011.nt.gz116447","sha1":"065ef67fc6e0c2980e1d0b11d1cbaa5e16451677","sha256":"f38e2a50d5b136ad2aa9ed86e33a6c81180138675f11036bcbfd01b7a2f2b64c","size":558097263,"status":"success"},{"md5":"b24df2c6d7a1adfdcbb29e80e9da5426","mimetype":"application/gzip","path":"output000012.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000012.nt.gz116447","sha1":"d1c1122c044a13abca1906157dd6c09dbcb0b6eb","sha256":"3bd311a9287d20c33b9442a7ce4d748196480a818ab944e1580138edbd7b383d","size":544001660,"status":"success"},{"md5":"9c16eac5b6aef2f5534c2db790290897","mimetype":"application/gzip","path":"output000013.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000013.nt.gz116447","sha1":"99f561a117fb1067e949205de9a5660a155dacf0","sha256":"dee6f2322dddbe750dcd4aa76776c8ff49ab02270a36b2de52afd662f65cdd89","size":545814184,"status":"success"},{"md5":"2b27e8088a0baf188cdeacaae0f44c8a","mimetype":"application/gzip","path":"output000014.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000014.nt.gz116447","sha1":"0a4ad759baf0ddad641e87db5697c687660b9732","sha256":"399d8337f0ff2605c03df03320f0efe51b2521d9c644c6faf44e446368e65c2c","size":522162410,"status":"success"},{"md5":"2c76044b634d4082feb7c8c2efdf4d1e","mimetype":"application/gzip","path":"output000015.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000015.nt.gz116447","sha1":"db3d979c5a1d2372c4a851c00be5006cc9d5337b","sha256":"511bfd2303db24d44d63a241e0f3e95475a45fdc13af78705b5fa8a9cc22ebe2","size":595370585,"status":"success"},{"md5":"e8992c69e204d475dc4b1c85af7edbf5","mimetype":"application/gzip","path":"output000016.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000016.nt.gz116447","sha1":"2b671b828a90035861714533193383ef1f1cd692","sha256":"986694f8c165b3153d4f9c4f0ac69443e72254c30629cc4b79328fae3dfcab45","size":572714851,"status":"success"},{"md5":"09e6ffc95a773b9825fbdf4c52557e28","mimetype":"application/gzip","path":"output000017.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000017.nt.gz116447","sha1":"82848a1b6f0f44326bab9458cf94ce422a4454e1","sha256":"18b522039429e13f9afa980e3f308266c249489dbb07fa1d2a12b8cb68ff6a14","size":584065733,"status":"success"},{"md5":"106e982e4ae4c35680623d0fce059ad0","mimetype":"application/gzip","path":"output000018.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000018.nt.gz116447","sha1":"3a28d2aadde5fb64034b59cefe414a6c3f8b59aa","sha256":"3deea8e0f9bb40af72d61c9e0c001d1d266cba6c42509140e4aa5f58f2a09e20","size":648195690,"status":"success"},{"md5":"5d1a97a5cf2e14ac735a891469d8d2a9","mimetype":"application/gzip","path":"output000019.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000019.nt.gz116447","sha1":"0c335c84ed04885caf243e6d091b8509e573b17d","sha256":"d6b554a3bcd06227dc6440034f2ff2d6fa9c53d5dbcaf4ed2cfe06557017af21","size":622451616,"status":"success"},{"md5":"a577c81db89eaba1b55906feb1d0c5b3","mimetype":"application/gzip","path":"output000020.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000020.nt.gz116447","sha1":"2b0b04ebb94c8139a5fad46c0638c28cc3e8a522","sha256":"4dc1096fb19e3f244942cd39acf404211ce804dfadc6eb8818e1355e4d51c447","size":531538569,"status":"success"},{"md5":"722378df486109b8744b6dbc0f37d8e1","mimetype":"application/gzip","path":"output000021.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000021.nt.gz116447","sha1":"db5c8a9586da3c4c12a239c8addf63b55239f3de","sha256":"84f45081aade62362f3c887ee22c119af1a97ece5d4d58f2048c14ab76d83ad5","size":670620972,"status":"success"},{"md5":"5afb3a7d07dc26d595752dbc15632d9e","mimetype":"application/gzip","path":"output000022.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000022.nt.gz116447","sha1":"df09f8e8d3b60bb5a2b49211f86c802bd0fc122c","sha256":"6146d40fbe39fa38ae2354f580f34dee6e96b6583ad3fcb0cb2247d8fd6a6161","size":692875331,"status":"success"},{"md5":"f277c4628716418f93e15e60f9fa5ff2","mimetype":"application/gzip","path":"output000023.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000023.nt.gz116447","sha1":"ceae85765619ad422fad9f8caa9c5f5ab2d3e503","sha256":"4f5f9013188c998938191d097f0c415c7e7ac5d547e663b35d588aeed1540229","size":591409951,"status":"success"},{"md5":"baed9ed2044d4e79ab425ef711ed26b7","mimetype":"application/gzip","path":"output000024.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000024.nt.gz116447","sha1":"f7c6419732dc0e560c7c10900ed029cb50a4ad8a","sha256":"9d43d7cf1e1637c2a0cbc91d7e1836caead6ae599455ef256c7ac508bfb6aceb","size":1015953857,"status":"success"},{"md5":"1d0cce567dbbe9827fc110597853a7f2","mimetype":"application/gzip","path":"output000025.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000025.nt.gz116447","sha1":"1263f632db8dcd7fa9ea4fae0998cbfef5b78fc8","sha256":"00f440915870e2fd14d73455bdf1fc9b4ce2980f7df61fd4f27daa45218130be","size":951808325,"status":"success"},{"md5":"fe653803af5d030d4c71d8befb087a47","mimetype":"application/gzip","path":"output000026.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000026.nt.gz116447","sha1":"d9c5f155ece63165c891baa8dc8fd7cfbf7258e9","sha256":"04abc8bca3ec3087d698112b0257f7ee9b139143ffca559b7036b3b4a05ff261","size":924019875,"status":"success"},{"md5":"68daeb5f21f4174da642278cbabe65f7","mimetype":"application/gzip","path":"output000027.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000027.nt.gz116447","sha1":"ee70a817698e3affdfaad79b4a96f9e1d9def533","sha256":"9add78364cf3bca70293f4a009eedb2a5d5f58f0ccbdf16b54ff2f6f40659ce9","size":1196589006,"status":"success"},{"md5":"e4151cfdf31ffedd33f561a4036f8d85","mimetype":"application/gzip","path":"output000028.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000028.nt.gz116447","sha1":"7743bdef659895c6fc89e798fb805d733960efea","sha256":"b93c6d37faf106a7f8e346cc7557598c3830d34ad14920e0897f81d777f66b8b","size":1212675011,"status":"success"},{"md5":"14a10c7945a22cc1d983036f51a3f9d7","mimetype":"application/gzip","path":"output000029.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000029.nt.gz116447","sha1":"2d8ccd691617e6c4ccdd324825328860bada2029","sha256":"fbfd3f8b0f0e2dad79ce7257ed87e03209723d444c74f7c079f945137b386b9d","size":1080662447,"status":"success"},{"md5":"4ea6700e0f9be60972c3d8b7f3512a11","mimetype":"application/gzip","path":"output000030.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000030.nt.gz116447","sha1":"8630c4dc1c4357030364b07d9962e2ec3d66aba9","sha256":"2b2c7b22e1165a3598f2436e617593caa1775a3adc6e6f498704dd3f837c1cbc","size":1241554328,"status":"success"},{"md5":"bffb1fdcd70cc11324b5b44c2ad7914d","mimetype":"application/gzip","path":"output000031.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000031.nt.gz116447","sha1":"695da0b6af3ccf92049217334b88a64a4338505c","sha256":"0dcfc417ff9ffc9d1b318d36da2b625c04c3bb26a1c1bbeadeada501cd4c1b55","size":1358309807,"status":"success"},{"md5":"2ac6a10f865be1aee22a55335fa439c8","mimetype":"application/gzip","path":"output000032.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000032.nt.gz116447","sha1":"41e9d984a14d6bb3c1b747f4357e34f30bc42d43","sha256":"0730e3899eb83efe973efc5a9545a909ec83e9e24d50b06050d730deb579e811","size":1387302890,"status":"success"},{"md5":"536d1c45bd8f9874ea36c69fb88497dd","mimetype":"application/gzip","path":"output000033.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000033.nt.gz116447","sha1":"0cf5fe3675537ee5778f42aeeaa27f539174e5a0","sha256":"a43053b1df0c6348f085e2fc0288b7f7e94e01e5889f1e0f0cdec4939af177f6","size":799368395,"status":"success"},{"md5":"3d490b853c5980cf6f5e3349999945d7","mimetype":"application/gzip","path":"output000034.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000034.nt.gz116447","sha1":"dce026eabeb6da64437a93b3f3a5236fe8c0e3e6","sha256":"ed404ae30a2aed01747e73bf9d026d7ed56369f393bdcc9e7c4dda98c1f2d001","size":629410568,"status":"success"},{"md5":"6362c3b82514723b898213653d033b99","mimetype":"application/gzip","path":"output000035.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000035.nt.gz116447","sha1":"f1c1227449a46bdb8b0553be764f28acddd32136","sha256":"bf6d7f4dfc5874420c38509cbbd6833386e0348bcd89334183397ae9d78b2b8b","size":612593415,"status":"success"},{"md5":"8c729d646ecde0ac4042d1a0d2ba10ac","mimetype":"application/gzip","path":"output000036.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000036.nt.gz116447","sha1":"9b9a77dfd09c146827b2ce795ba02554f46855ba","sha256":"fc67e5568e49c03189b42bfc743c9a1163e8f4e8286a31900ace0e58da17c0e8","size":583156139,"status":"success"},{"md5":"9c30e943f217fb9c348ce2fc9e0525d3","mimetype":"application/gzip","path":"output000037.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000037.nt.gz116447","sha1":"a0dc52155b68525887420df7fc140d3589af0c9d","sha256":"10ff94628013ec65d227c50ad345c1dca706d86c9519ea4fa5abbdd1f46c5070","size":610130381,"status":"success"},{"md5":"9b852ddb6c848c8d68a2c0efd221a20b","mimetype":"application/gzip","path":"output000038.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000038.nt.gz116447","sha1":"2412f6d35bf59bce7379e00f3e34c36734042043","sha256":"0960fc30829da9a4068eb177277b5352799b85b3e2f4b5508d93affcff5af4a1","size":597187150,"status":"success"},{"md5":"021697ab9648fb3756aeea8e61464a64","mimetype":"application/gzip","path":"output000039.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000039.nt.gz116447","sha1":"79192538a7b68b4d35048af01141e335df08a174","sha256":"84f58c795aaec46c51769cd3d2d7ce10bfc74bfefd63b3ca8299a448ac06bbbe","size":634799547,"status":"success"},{"md5":"812e81027d451b6662c5b1a5611091d7","mimetype":"application/gzip","path":"output000040.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000040.nt.gz116447","sha1":"d4ad2b772a1b4d59c6a5cf131fe49d7832ba6409","sha256":"66276c8b5e28d295adb545d0bac026cd4e61d130be21d901802959ae8a6e9d40","size":646356377,"status":"success"},{"md5":"cf00f05ba3f0c89034934538828e6aeb","mimetype":"application/gzip","path":"output000041.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000041.nt.gz116447","sha1":"7b0da07e434267a83848d2449e805bd3d8d01f55","sha256":"ff9dc0d1acf3591e4475526f4cea0ad1771f52a6a460133289b3f20916bf9b91","size":665040394,"status":"success"},{"md5":"507487efcd9ab04e15a116a97a495712","mimetype":"application/gzip","path":"output000042.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000042.nt.gz116447","sha1":"f87dc1b301996318060761dea6a3b74204abb96b","sha256":"a1dfdc4181400815cfd4b7bffe4d658fcc5138d9135f95f05613318e942c5847","size":660921038,"status":"success"},{"md5":"5b20c6db11e7122c459b9d4116d051ed","mimetype":"application/gzip","path":"output000043.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000043.nt.gz116447","sha1":"f271fd0380a394367cefeb6b99681e22d8920ef6","sha256":"9ce604b906a877d6ded6621ab416a8878beabfccf2a744be25c71ed273e64e52","size":707476090,"status":"success"},{"md5":"28284f7123806c09491f314e9489eb7a","mimetype":"application/gzip","path":"output000044.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000044.nt.gz116447","sha1":"46510b869c637459f88c1f151834c9e8486a26d2","sha256":"080157d1ea36ddd7222c9c421c9cc76d3c367cb7f9d858d7f18e5c2841e1fb56","size":731222194,"status":"success"},{"md5":"5f9e490c1cf2e4336ce179417925963b","mimetype":"application/gzip","path":"output000045.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000045.nt.gz116447","sha1":"e22abcaa13fb1268a6ab4430b05b51c41bb6bcce","sha256":"53864ade4c319111ce5269c7c027b26c8ceacb474429cf850a9836802bda860d","size":714523439,"status":"success"},{"md5":"df8b569ddcb0511c66d4454aa9f60f27","mimetype":"application/gzip","path":"output000046.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000046.nt.gz116447","sha1":"cbb6b848705440d3da11586e10a9438de9251af9","sha256":"267d0e790a56035ab2681d8846d13adab56af2407b9a1db71a2d521b147a16dd","size":654550072,"status":"success"},{"md5":"0a542c46ed27485faa44fb42e6d4452f","mimetype":"application/gzip","path":"output000047.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000047.nt.gz116447","sha1":"adf253d9a98a64731860550391fd993762f7e2bc","sha256":"8aaef1c515b2997c854591c31427df907fc8b77a95d13b82b37f288416b6ad2e","size":661374950,"status":"success"},{"md5":"5a1e64b43040668f6d6b9fb4dd510451","mimetype":"application/gzip","path":"output000048.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000048.nt.gz116447","sha1":"d49f48e5398b59af5fc02edf1ef92d2b12f6e10c","sha256":"dc22f45e6ef6230a1603d16b735b9918edbcd3e92984e8cfb107eabbf69bfd72","size":619824072,"status":"success"},{"md5":"5bf57aa135fd243ca5ecc83b42df7e47","mimetype":"application/gzip","path":"output000049.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000049.nt.gz116447","sha1":"31aaf8a77deacb1445d4761bc0484d8e17837caa","sha256":"9d45bbcd0e0920c74ea4ab8dd612c70abcba07535a84e24d0af068ad62ad2f05","size":579960996,"status":"success"},{"md5":"f6d254c41bd0981eb9ccb5628eaff34f","mimetype":"application/gzip","path":"output000050.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000050.nt.gz116447","sha1":"38d1f9553eafd86e895a90fcb12cfe54f90058b4","sha256":"1908522e95ca03c686cc6a60948d9aec0aeeba992a8243f220014f11c666516b","size":577983399,"status":"success"},{"md5":"ca7526cb5c694d218cddc7284b644b4c","mimetype":"application/gzip","path":"output000051.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000051.nt.gz116447","sha1":"460d8e21f0c7def5e845cdd66e2cf0cc09a8da61","sha256":"c0f26e5ad694d0001cd86f76f09b42e8f0e46b396c5a44058c2d9d64875e237b","size":581178846,"status":"success"},{"md5":"dcaeeb3cf9a02008edc80e8f5dd9948f","mimetype":"application/gzip","path":"output000052.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000052.nt.gz116447","sha1":"12df33fa167fa677bec413068b6bc5699b0b7da9","sha256":"d1b43d512e5386394ee9581b2293ec2c228b5163375ab7c40c66a1a1f37fb512","size":568578597,"status":"success"},{"md5":"ddb6e623ab5becb9ac90f209f2cdda7f","mimetype":"application/gzip","path":"output000053.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000053.nt.gz116447","sha1":"98ab56e789653577f9fe113ad5ad720d6c37efed","sha256":"319fbb7bff99cac0ab90d947702e648f4d277451602fd6ed1cfc146f04654365","size":565813766,"status":"success"},{"md5":"025e2dcd9f51494c1902e622ec35cbfb","mimetype":"application/gzip","path":"output000054.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000054.nt.gz116447","sha1":"2443239dd213290d3dbe2d8e7e4d07a9f8d35287","sha256":"cc87450dd82825a3ff260aec51852870d6e49677c4e519399fb0c08afa5050e8","size":566668804,"status":"success"},{"md5":"c9c61d6b503f95659c9a16e669dcf834","mimetype":"application/gzip","path":"output000055.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000055.nt.gz116447","sha1":"af3ca4936a2e054aa939102a20424ffe35327a86","sha256":"ab7e68327b9a2ae9638ca3c8578066b1b30778f1fe235e5d1d56b976769395fc","size":564691218,"status":"success"},{"md5":"0b3f000549bac88dbea8a316baf1ac58","mimetype":"application/gzip","path":"output000056.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000056.nt.gz116447","sha1":"837b2e7187e514c7d9ee7f3b4f8215be004c9ba0","sha256":"6e7968f6caca0e3f644a9b44908ff1f55d602c21d33098e89a6b27ba9e447fa1","size":549186972,"status":"success"},{"md5":"f35564de7f9ea8115ba3dd66a9e6575a","mimetype":"application/gzip","path":"output000057.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000057.nt.gz116447","sha1":"e4d9a5b6a660b65664771323813b5f1d4a11b8d5","sha256":"06e6e5ad2d2b731f0f2e7015275b0467b5a828eb211028d5481542ccf812ae1e","size":517724089,"status":"success"},{"md5":"c1230d3a64cc4c1ee10d7fa3cbfc274e","mimetype":"application/gzip","path":"output000058.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000058.nt.gz116447","sha1":"e04f65fda1ba14fcd03ec4f1f74390ff674085db","sha256":"8198a5af053ff581463b1c32e574f9caf9c0f9f631d486b05c9460fbc18e070f","size":529662993,"status":"success"},{"md5":"1a8eabfcfba355d06513b98885d3da86","mimetype":"application/gzip","path":"output000059.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000059.nt.gz116447","sha1":"7d07f366bdab74e53f31ad86554b7217b0355531","sha256":"59d59b313304ea76139ae72d548eb75404c65d2fdd7968e4816e19cf0e905ea7","size":528070551,"status":"success"},{"md5":"bac1f20a352193a78c0240510528465d","mimetype":"application/gzip","path":"output000060.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000060.nt.gz116447","sha1":"92e7271e2cb1383476797cb7cc6d37aa79b72783","sha256":"35b9cb721d062433b6090985a980daa7194acded222f1652f0779a5033d32e95","size":511463487,"status":"success"},{"md5":"9d0dcb49cfda1c3d24f36e397f3a51b7","mimetype":"application/gzip","path":"output000061.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000061.nt.gz116447","sha1":"c8881f0d18addfeef4b84d365e58d386c18e079c","sha256":"cfdd471ce43839b6ff63d5526f31c7e7faad95fa078d51ea29cf93333dbf70c1","size":901247568,"status":"success"},{"md5":"a394c8057cb0386302f316a837c7e63f","mimetype":"application/gzip","path":"output000062.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000062.nt.gz116447","sha1":"3fa261860a28eb3829a9f836a0b4eeb1d6e047ee","sha256":"0f0e32f0c5895e79d5890f4d07cf90c1d7001182f689090452ca24dc4e33f346","size":877130699,"status":"success"},{"md5":"c9c77b50ca7ebbba6f40528ca0fd848e","mimetype":"application/gzip","path":"output000063.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000063.nt.gz116447","sha1":"e31e65b75012d44c2a6dbcfdc8cfabb5f6f42ef6","sha256":"b2e78557ae304a2e91d564c8b7fb19f719120d1f51e1e43c67884c06710f2e63","size":767862916,"status":"success"},{"md5":"6a042e4b2fe96363bb4f659c79d9cf99","mimetype":"application/gzip","path":"output000064.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000064.nt.gz116447","sha1":"b422a50a0d1828137503b84df7c77f6f52511da0","sha256":"b9414f047777384a21b76ae20ba2aa72989a8db69d7f61aa790c530eda45aa06","size":809982357,"status":"success"},{"md5":"803ee74088c97c7951465a6686097f00","mimetype":"application/gzip","path":"output000065.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000065.nt.gz116447","sha1":"8b00a7567db56763dfcb3249df14fb8c4e05597b","sha256":"eb7dad7947d4a7f79d6c3fdbafcb71512b3ff69e712e693db84ec94cf4ed81a6","size":818194052,"status":"success"},{"md5":"96ddc26f58ef4ab1317a7fe7b9add2c6","mimetype":"application/gzip","path":"output000066.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000066.nt.gz116447","sha1":"1b48c98c8d9238efb594f5b175c6aaf78db8e054","sha256":"df3ebb9b1282319e92eda526b8a1d81c2c71c2c4ddeb0706fcdaaa5263578440","size":622864555,"status":"success"},{"md5":"60d3ab3d0b484d2e1f3ad92f1210d5f9","mimetype":"application/gzip","path":"output000067.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000067.nt.gz116447","sha1":"29298e02650becb96e4a413acc7fc78b42a42e52","sha256":"626a72eadb4187bed812f8e93b454319aedb6d4d4c8e01259a1090dc5852dd08","size":482322117,"status":"success"},{"md5":"e9df3613d2f8309b3accbbeb623cb10b","mimetype":"application/gzip","path":"output000068.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000068.nt.gz116447","sha1":"7169c827fac3f5f4a0684a90b11969ee93e75033","sha256":"5d9aad35937da37d7d057ce428f1cb5c0972efdbf3b1a8d97ca87886f6cb599b","size":513600457,"status":"success"},{"md5":"2b8b424414b87f77b7103556d520e26d","mimetype":"application/gzip","path":"output000069.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000069.nt.gz116447","sha1":"dfe2f4b03b3ffc0647166e294d0868defd6d9ebc","sha256":"db30f3ffb13043f6ab37e66e1e885ba188cee55fee770a183ee7fb8ff3bf19c0","size":511817368,"status":"success"},{"md5":"9b2da059a8ae7fa5e0fcb5118a005027","mimetype":"application/gzip","path":"output000070.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000070.nt.gz116447","sha1":"2fb9b750e4a09a3d4fb7b7ce7e71350998875db9","sha256":"15138d2375a405c68db89ee04af53f17dc8dc698a4cde3b211ba595fad8a4e5b","size":510886315,"status":"success"},{"md5":"c0f631efd33bbad1baac95b9a986ba5d","mimetype":"application/gzip","path":"output000071.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000071.nt.gz116447","sha1":"2114082075b5b8363f1ae855d4297d1066465325","sha256":"6eea789e98d497a4aaf003c549366ab6b962714f6fbc969b5f4d917d943c733a","size":511318353,"status":"success"},{"md5":"dd04f27c5022ea3313fece2904d287a4","mimetype":"application/gzip","path":"output000072.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000072.nt.gz116447","sha1":"9a74d52452d839030ef064c80abbf8565bdee18d","sha256":"a0aedc770bf9e42811cbf5a32abc22f6a17712309293799ad6a8d0de2dc25a7f","size":503429197,"status":"success"},{"md5":"0285c4d58add4265be6c7f1bacd8c1a0","mimetype":"application/gzip","path":"output000073.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000073.nt.gz116447","sha1":"b033a3f737ad466326c3b3db1a706cbabf99ccfb","sha256":"d800523b6541939f5c6dd5abd09fa4517a689c12996ef04cb3acf846253581bf","size":443531221,"status":"success"},{"md5":"9966ba5a5520340e5538acda913f0f13","mimetype":"application/gzip","path":"output000074.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000074.nt.gz116447","sha1":"ecfcde443f7dc9450f1650f725ee2cbbd1c546bb","sha256":"20e187a954fab92e404c50b39ccd1c3c675b5c25e1f6d807fa572783bd442b71","size":373829527,"status":"success"},{"md5":"edc66cf01feb2947fe441d4fcf834c14","mimetype":"application/gzip","path":"output000075.nt.gz116447","platform_url":"https://zenodo.org/api/files/f6ed487b-a16c-473f-9ab2-addaf048ed0d/output000075.nt.gz116447","sha1":"8854c77192970e277e5663aeb0207b7d92aa1de9","sha256":"817f800bc87bb3f5cdee78820cf416078198999e4b1ddf58700c9d029c13d2af","size":131730741,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1321718","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1321718","ext_ids":{"doi":"10.5281/zenodo.1321718"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"c3zomgfslvdpjam3g62bnfnphy"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1321718","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20210508144055","terminal_sha1hex":"3ba44579236d2fae46ca3181585c80936273525e","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1321718"},"total_size":49095370137} +{"archiveorg_item_name":"zenodo.org-3322275","cdx":{"datetime":"20200226161615","mimetype":"text/html","sha1b32":"7G3YIU4CWSX3272LJ44EFVWQ5PZMYTKS","sha1hex":"f9b7845382b4afbd7f4b4f3842d6d0ebf2cc4d52","status_code":200,"surt":"org,zenodo)/record/3322275","url":"https://zenodo.org/record/3322275","warc_csize":10123,"warc_offset":104769917,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200226155702759-28904-28969-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200226161309864-28918-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":30,"file_meta":{"md5hex":"b3d1ea1932ad27c2f736647b9fa0855c","mimetype":"text/html","sha1hex":"f9b7845382b4afbd7f4b4f3842d6d0ebf2cc4d52","sha256hex":"5446e772296ae001a6da9d003a4868ed5f099ebe87a8ab8a411cec9b4a6cc4be","size_bytes":59503},"hit":true,"hops":["https://doi.org/10.5281/zenodo.3322275","https://zenodo.org/record/3322275"],"html_biblio":{"container_name":"Zenodo","contrib_names":["SXS Collaboration"],"doi":"10.5281/zenodo.3322275","release_date":"2019-07-11","title":"Binary black-hole simulation SXS:BBH:0861"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"cf88d4f218c67de7589d9549f59b7418","mimetype":"text/plain","path":"common-metadata.txt","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/common-metadata.txt","sha1":"ec5924d99a084a8b66b1785f2eae8054e93d7921","sha256":"75d892d2057ff55c401f7737ba9512292ed9bee832730f4b2caf296388c3ca31","size":101,"status":"success"},{"md5":"de551139c2b41e4f767ff130e7895f8c","mimetype":"application/gzip","path":"EvID/ID_Files.tgz","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/EvID/ID_Files.tgz","sha1":"e1ee4f611b9281c2ab95c11f671bfc52e7f0d70f","sha256":"2bfc2d62bdb5dd6aa0653cd4984c3345634e6fb9a98a123f99463ec53a888d8c","size":14972693,"status":"success"},{"md5":"7531ba690142c55efac710fa02424032","mimetype":"text/plain","path":"EvID/ID_Params.perl","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/EvID/ID_Params.perl","sha1":"b0da8ebf3b1fa95fe847ed510f985c26c73191a5","sha256":"a451ed842b4a3cab4a1ff1ed718c9b37866b43b26a8b2dad1392f157c7e9d21b","size":1565,"status":"success"},{"md5":"9ddf543a877ee5230a178248e00c1de3","mimetype":"application/x-hdf","path":"Lev1/Horizons.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/Horizons.h5","sha1":"084a71d8fd269fddb44eaba2dcdc2ebce379d056","sha256":"5b1097fe8f5630a6f024408a20a5c2c12578b9a09700385feb1f0abc9c859c98","size":2535186,"status":"success"},{"md5":"890adf080789332afee982b599224f46","mimetype":"text/plain","path":"Lev1/metadata.json","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/metadata.json","sha1":"da9d1165488f53d0555ce0dac60e25d177635f2d","sha256":"783d15bb46376063d3c41c881c9d7666c25237e0f75737e335d2f46deadbd2bb","size":3811,"status":"success"},{"md5":"3db748914be6abef97ed3d48446a77b2","mimetype":"text/plain","path":"Lev1/metadata.txt","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/metadata.txt","sha1":"1c2709e49cb35404de3be1086bfbfb7cefa64957","sha256":"a49ba31cc4bf60e61a47b6c4b352f1a0fa7285a9062e7553674d5e9fbcd6e0a8","size":4813,"status":"success"},{"md5":"2864373e8e05d112153d257e45a85b4f","mimetype":"application/x-hdf","path":"Lev1/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rh_FiniteRadii_CodeUnits.h5","sha1":"d1e6bcc0015c96849b89a18f1b135cae1c0fc9f7","sha256":"cd3f15cf9e7a23cec6e26d8f847dd5592c778d0fc6264b0b10463b178aa2080d","size":499571259,"status":"success"},{"md5":"4ff16149678e49dfb141b84a18771af7","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"472ecd3bbec712abb4f39945224c5ae5a6b10054","sha256":"89ef1152f411e403140556fc6dbd6be1d1cf75d232b6b912449c9d28cd6582e3","size":89353957,"status":"success"},{"md5":"16581abf8dbd9392f0294ca04bbc0c66","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"11ce12aa5f4ffbce69076dc338556bbd55562549","sha256":"3346745ea2a4d7e5bc7a54d944a9b55bd9fcc5031f12d6f470e58412aae1e06e","size":92970849,"status":"success"},{"md5":"43b5c7acde428464cb806da4ac8cf18f","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"bd1ddf1ae6772ae770072f2d0c3df4aa399ca134","sha256":"db3b8ac1e694b261ae9e6bdf93baa13a7c02231ff5f38f9b1643440a6c38fe1b","size":89717666,"status":"success"},{"md5":"4d52f8a902119559da42372f89af3956","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"29c5922f22ddbe429888f75bfaa8a8b70272e5ad","sha256":"139a23355dca667a8f270781f7f682706fd00d50a1f4eb581e10636e40a4ea20","size":92546556,"status":"success"},{"md5":"3f161a8a93469eedb389085db9dbd786","mimetype":"application/x-hdf","path":"Lev1/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev1/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"af891f1ca9721ebd7a2e339d9a3dffecb20e7aa1","sha256":"3f797f1bcd83952be979b8e4315b1826a01d487f1a0e5e322ac5b411631bd58a","size":493260879,"status":"success"},{"md5":"62706ffe0791af9536bb2fb27d1aa0d1","mimetype":"application/x-hdf","path":"Lev2/Horizons.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/Horizons.h5","sha1":"e76b1014615f2595c262a101a9c55c1b8e21fed4","sha256":"12f3acbf04bc76a0b16b73ed593866728266cbef03094f1debef4a65cd1a17f3","size":2478298,"status":"success"},{"md5":"1e4e8371627b6237d92ab05b5971b5e0","mimetype":"text/plain","path":"Lev2/metadata.json","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/metadata.json","sha1":"49352afc1c73c00d50a4ba447cb112da993ef072","sha256":"a7fd0b7565a7bd2d974ca31b6a47795a9dc4c750091950ab3496bc70346bcc40","size":3816,"status":"success"},{"md5":"fdfc7d2329da1506324450f7c923f1c8","mimetype":"text/plain","path":"Lev2/metadata.txt","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/metadata.txt","sha1":"0058441db7a8ff5f5caf1556821d56a1969bcacf","sha256":"dbf767c65c8435d984de4635eb447dce48949b3f675a1a575d46ed277eea5151","size":4818,"status":"success"},{"md5":"fde32b5834f286a4c9b45120624a728e","mimetype":"application/x-hdf","path":"Lev2/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rh_FiniteRadii_CodeUnits.h5","sha1":"0335d75123fe115bda8e0ef7bbf7993a139ca261","sha256":"e2c6d23523d2cef94dfe553e1371bb1808f169675987be023af180d7034e6be3","size":492551774,"status":"success"},{"md5":"688c916086ec8dc17b29baca6a60e55d","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"3f9df79d62aca6c17c367493d27ee5728c44aecc","sha256":"fc6256a87f6e4ff3a07691aaf68751efd44edcaedfcc1ce2fe912689f9f533af","size":89240727,"status":"success"},{"md5":"d4ecde6cfc3d3838a297f5c2736a696b","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"ee6bb78b49b654c7d38dd327345ecfb4cdcf87e0","sha256":"8826b882f84f8bddbb65ee458e06567bf44c517deadb2573a1ee390eb08dccf5","size":92228040,"status":"success"},{"md5":"e05ff912ea215d1201c6cfb155d42e10","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"d94db472ae15d3d9695f2a6f807f92668eed82c9","sha256":"36f64a93c6ce173164ff185ec0b7a718515c6172af78a17bd70c3de35240471f","size":89860033,"status":"success"},{"md5":"1b6ee0e39c8fa541469e40f3e0f79360","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"a9928830a8f2e1a95fb00f583f89b102b31d72eb","sha256":"78c38ced913b09ef8b06e0a456a16ad45349d4194b02d9ec2fcee612f9c184f3","size":92167283,"status":"success"},{"md5":"a4ede1774f9d222a7ab21d90421c7ccd","mimetype":"application/x-hdf","path":"Lev2/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev2/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"b7f7bb18cf4deb09b3be3882ca8684ef8e3d5b04","sha256":"70cc71e74c4115f5039e2e2363675a87a2eb6ad028547436dea4bd4ff986dfa5","size":492738522,"status":"success"},{"md5":"2387af1481196e47d60727b94997ff69","mimetype":"application/x-hdf","path":"Lev3/Horizons.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/Horizons.h5","sha1":"392599a0bd0560706a3a61f088f8747d94b0a350","sha256":"639419375979d37425cf15b54ba8447445fef09837024cf58cd34a6f941c1af7","size":2499796,"status":"success"},{"md5":"5d7b03db3bd3ce9cd5bd896915bf73fd","mimetype":"text/plain","path":"Lev3/metadata.json","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/metadata.json","sha1":"ecfcd1cc6c3c270b3acd5f5249b2e6d433200b81","sha256":"70e49d9d8bdcf184fe2c22a3acf413853cfcdab43d74a43dc0ad563fc097f4ae","size":3808,"status":"success"},{"md5":"63839178515edb22735be791653f4fc6","mimetype":"text/plain","path":"Lev3/metadata.txt","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/metadata.txt","sha1":"4805a1edd178080a2dfc7c89cd23c852eebb4a10","sha256":"25a749e631abcf33c00d9e113eb6b4b85f34081389153c691efafbd6bfc7ad4d","size":4813,"status":"success"},{"md5":"0a2325a3edbea8090449c57bb1f25db0","mimetype":"application/x-hdf","path":"Lev3/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rh_FiniteRadii_CodeUnits.h5","sha1":"c1de0f422afc55ffc4f8691f3c7e3c8d30e7f2a8","sha256":"8a0c00c7c2804b5bc3e559d069942e96e86fe15e4d8ecfb7ced9ce569eb1f41b","size":494518666,"status":"success"},{"md5":"be0e29224d5493956b3851cae8e86da5","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"6813dc693ed5732bb718fbc360b85643f959d1a8","sha256":"f427e58922938933b7061856e4e2e1ebe243ce7b03f8fb9f1e4d2b9119b2b2e2","size":89646561,"status":"success"},{"md5":"3d39b49786545d32bbf3b57a93263a40","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"c64cc9abc9be4bb09e97208660cf12d2b8ca08b6","sha256":"1370e1675cb7d3cf61125ff898df07ad3795776019ea7972aee3e7913c3561ed","size":92707870,"status":"success"},{"md5":"fe6bff535ff3dbf97e83effd15746c14","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"61a978441b0166a9dcca8aa2e46c6e31672e5726","sha256":"cb591278d64fbc4712202c274d0d2aa2f3adc1341eb25451e4fda6ed8a92b9c7","size":90344612,"status":"success"},{"md5":"6a87671431b7704373bfca9e8161ed76","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"e7c73b14ada7cc2d47476f45d4826c1091ba66ef","sha256":"5110801b2aeedb740488ac64f66358ec4f5a8bbc451533a4a60b3adeca940d0d","size":92567703,"status":"success"},{"md5":"74d11fe4a769fa17ba2e65bfda132b1f","mimetype":"application/x-hdf","path":"Lev3/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/78c87a79-1fc8-46f0-83fa-66c206d1e9eb/Lev3/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"3ed2a35db33746dc81e9f4027244079f533badae","sha256":"fada56b4e0de3b2a7c74ecafa9d7174d788f671df0f6ee74ea9e2040a89de041","size":494660462,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"3322275","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.3322275","ext_ids":{"doi":"10.5281/zenodo.3322275"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"awggyedr7jan7e4q5eqx6ffx2y"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.3322275","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200226161615","terminal_sha1hex":"f9b7845382b4afbd7f4b4f3842d6d0ebf2cc4d52","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/3322275"},"total_size":4083166937} +{"archiveorg_item_name":"cellimagelibrary.figshare.com-8184362-v1","cdx":{"datetime":"20200805074834","mimetype":"text/html","sha1b32":"NUYH23NPDPMJXKLBIUCQBOUIVJXW3RET","sha1hex":"6d307d6daf1bd89ba961450500ba88aa6f6dc493","status_code":200,"surt":"com,figshare,cellimagelibrary)/articles/ccdb_6736_jpg/8184362/1","url":"https://cellimagelibrary.figshare.com/articles/CCDB_6736_jpg/8184362/1","warc_csize":6603,"warc_offset":249510441,"warc_path":"CC-MAIN-2020-34-1596439735916.91-0024/CC-MAIN-20200805065524-20200805095524-00495.warc.gz"},"file_count":2,"file_meta":{"md5hex":"19eeb93c2f0cf5322dd2caeb49bae990","mimetype":"text/html","sha1hex":"6d307d6daf1bd89ba961450500ba88aa6f6dc493","sha256hex":"73bded833bc6f38f83ce3bb4c903e10b242709b3233a536222f73993d3b17ddc","size_bytes":28130},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.8184362.v1","https://cellimagelibrary.figshare.com/articles/CCDB_6736_jpg/8184362/1"],"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"bb71b868a02a4868eb5d37dc35e33bd2","mimetype":"image/jpeg","path":"CCDB_6736.jpg","platform_url":"https://ndownloader.figshare.com/files/15253451","sha1":"8193b3c09fe3986cdbf3396a89f03db5668747cb","sha256":"dac558a07344492e6c572b1f7db1c88ead0f2847b21cc0d03007af03390756ab","size":94333,"status":"success"},{"md5":"381f3e0e2f02c754eded647652e3e6b8","mimetype":"application/octet-stream","path":"astrosoma-g4s5_z_-208.0.out_single_bin8","platform_url":"https://ndownloader.figshare.com/files/15253454","sha1":"7399505cc412005c45fd7f3b114e1b5f6fbb2565","sha256":"cf29e94b948735373c06e89459fc20ede0a69637924e1c05478eff3d550828b5","size":173281024,"status":"success"}],"platform_base_url":null,"platform_domain":"cellimagelibrary.figshare.com","platform_id":"8184362","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.8184362.v1","ext_ids":{"doi":"10.6084/m9.figshare.8184362.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"dmbqpfhpc5cgbcmky22tvhhfd4"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.8184362.v1","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200805074834","terminal_sha1hex":"6d307d6daf1bd89ba961450500ba88aa6f6dc493","terminal_status_code":200,"terminal_url":"https://cellimagelibrary.figshare.com/articles/CCDB_6736_jpg/8184362/1"},"total_size":173375357} +{"archiveorg_item_name":"zenodo.org-1129411","cdx":{"datetime":"20200227190423","mimetype":"text/html","sha1b32":"ZNN7B6SDZXZEALKY7PHG4J5BGHALCZUF","sha1hex":"cb5bf0fa43cdf2402d58fbce6e27a131c0b16685","status_code":200,"surt":"org,zenodo)/record/1129411","url":"https://zenodo.org/record/1129411","warc_csize":8559,"warc_offset":100872864,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200227175350369-31156-31221-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200227190222039-31221-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":8,"file_meta":{"md5hex":"4fefa8acb1a3d4149bb4d5fa95a95206","mimetype":"text/html","sha1hex":"cb5bf0fa43cdf2402d58fbce6e27a131c0b16685","sha256hex":"3b1e7046e33ee07deba6e6444e7110412ce448b8a454d39f326a4a45269edc73","size_bytes":37177},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1129411","https://zenodo.org/record/1129411"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Thomas Piggot"],"doi":"10.5281/zenodo.1129411","release_date":"2017-12-24","title":"CHARMM36 DOPS simulations (versions 1 and 2) 303 K 1.0 nm LJ switching"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"7c6abbc77283a3ffa780e1d47a552eaf","mimetype":"application/octet-stream","path":"for-md-CHARMM36_DOPS_303K_v1.tpr","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/for-md-CHARMM36_DOPS_303K_v1.tpr","sha1":"316a64f5998bd7d992ad0cdb264a7eb154dde78a","sha256":"e5fb48a4ced9162ad9e00bb35e89d56880dc6ef31eafea56dcb9521690b28c32","size":1015016,"status":"success"},{"md5":"9d6b80d955cee9886efe0d961978a654","mimetype":"application/octet-stream","path":"for-md-CHARMM36_DOPS_303K_v2.tpr","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/for-md-CHARMM36_DOPS_303K_v2.tpr","sha1":"96a292e484ec84f102f76c87584e373a35688cbe","sha256":"f996238108fe3bb2fafcedbd28328f8637a06982a8e8774603db124689d3a861","size":1015016,"status":"success"},{"md5":"abbe19da1d4e7fbf9aae616a0e0ea0f4","mimetype":"application/octet-stream","path":"md-CHARMM36_10A-switch_DOPS_v1_400-500ns.edr","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_10A-switch_DOPS_v1_400-500ns.edr","sha1":"e26fc751c35f13bc0e8481b5283672a0f9ffc645","sha256":"75b89f1af04dc7c6fa23a02ab983e6bf5cfd1321482e846fd8ffd49a537c794a","size":2497720,"status":"success"},{"md5":"b55d2462d749e24527351748796a9e0f","mimetype":"application/octet-stream","path":"md-CHARMM36_10A-switch_DOPS_v1_400-500ns_skip10.xtc","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_10A-switch_DOPS_v1_400-500ns_skip10.xtc","sha1":"0f44a3751fd7025b7c3ad4a1898aeed7a6993624","sha256":"494716cfd093e21d3c29902f1cb8d630cc5bda69d76620caed8f431a69e7f438","size":226054228,"status":"success"},{"md5":"4f284b27354b668c4c0432ca672b90df","mimetype":"application/octet-stream","path":"md-CHARMM36_10A-switch_DOPS_v2_400-500ns.edr","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_10A-switch_DOPS_v2_400-500ns.edr","sha1":"d19b6cf152caba51d22910321971786f50cf4edc","sha256":"15cb45eb2acf81c9ce44ddbfd621d06970012edb6445e605203446f1763567ac","size":2497720,"status":"success"},{"md5":"6c068d1645d54d6218a29ee0840c3ea0","mimetype":"application/octet-stream","path":"md-CHARMM36_10A-switch_DOPS_v2_400-500ns_skip10.xtc","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_10A-switch_DOPS_v2_400-500ns_skip10.xtc","sha1":"b5db545685561c065703b359b454484f53c01fe4","sha256":"050a9a518571f6914f83d281361c80065925f82645d8dca6cbc41a27641b1986","size":226069900,"status":"success"},{"md5":"63ee1ffec14ba058c5cb8b5be2b113a6","mimetype":"text/plain","path":"md-CHARMM36_DOPS_303K_v1.gro","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_DOPS_303K_v1.gro","sha1":"cc4e76ec154275ef339df5010a2e379d19ff62f9","sha256":"efcdc96a86af1f864fa993197344214d3365b89a297ec3885345b39c40b01669","size":2093227,"status":"success"},{"md5":"6f50dd9e9852eef254e67bfdfecbe139","mimetype":"text/plain","path":"md-CHARMM36_DOPS_303K_v2.gro","platform_url":"https://zenodo.org/api/files/0580bfa0-8cd7-4004-b0c4-4dea1988b253/md-CHARMM36_DOPS_303K_v2.gro","sha1":"f17013e3412124e5fcb3ae8055fc05e99feac2e0","sha256":"77113afd2134bfa4475b4832a6e25aa7f46b5f30d0b83131422b4aa5849961b6","size":2093227,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1129411","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1129411","ext_ids":{"doi":"10.5281/zenodo.1129411"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"f6essm3kwfefffzwj4nr3awp34"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1129411","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200227190423","terminal_sha1hex":"cb5bf0fa43cdf2402d58fbce6e27a131c0b16685","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1129411"},"total_size":463336054} +{"archiveorg_item_name":"zenodo.org-1003567","cdx":{"datetime":"20200216111242","mimetype":"text/html","sha1b32":"G4NUB3BK3V3T66VXQFFOH6TS6UMTUNX6","sha1hex":"371b40ec2add773f7ab7814ae3fa72f5193a36fe","status_code":200,"surt":"org,zenodo)/record/1003567","url":"https://zenodo.org/record/1003567","warc_csize":9715,"warc_offset":92161513,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200216105940694-15850-15916-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200216111137320-15863-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"af0ce9b0fabdfb1b627925da4b16e870","mimetype":"text/html","sha1hex":"371b40ec2add773f7ab7814ae3fa72f5193a36fe","sha256hex":"ff6c7985c7420284789079fb01636df4b15cfbf2ce42bbbc4af3135c64042b17","size_bytes":40113},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1003566","https://zenodo.org/record/1003567"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Javer, Avelino","Currie, Michael","Hokanson, Jim","Lee, Chee Wai","Li, Kezhi","Yemini, Eviatar","Grundy, Laura J","Li, Chris","Ch'ng, Quee-Lim","Schafer, William R","Kerr, Rex","Brown, André EX"],"doi":"10.5281/zenodo.1003567","release_date":"2017-10-20","title":"N2 Schafer Lab N2 (Bristol, UK) | 2011-10-06T12:52:32+01:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"dc31438acca67aa278c55660801441df","mimetype":"application/x-hdf","path":"N2 on food L_2011_10_06__12_52_32___6___7_features.hdf5","platform_url":"https://zenodo.org/api/files/30a02c2f-5d73-4137-9d77-4f0b74ccdbef/N2%20on%20food%20L_2011_10_06__12_52_32___6___7_features.hdf5","sha1":"1072e87d46a4b82cfe1980b187f64989166a193a","sha256":"1b6e8ebd3e002fd3a7a69d15080870436585714cd0aab9424f29f5e582ae0556","size":21091104,"status":"success"},{"md5":"23e4d569ed2bba1c7778cdf799569237","mimetype":"application/x-hdf","path":"N2 on food L_2011_10_06__12_52_32___6___7.hdf5","platform_url":"https://zenodo.org/api/files/30a02c2f-5d73-4137-9d77-4f0b74ccdbef/N2%20on%20food%20L_2011_10_06__12_52_32___6___7.hdf5","sha1":"fb567a9f29822b06d63cd126f4f0e1afb7b0039c","sha256":"92f36fc1e5fe6a1701320254d7c4e8901b5954fdfa9dc41c2635f8d2dc9c986d","size":263084882,"status":"success"},{"md5":"9f1471ada97cad206e5619e87f0cb49d","mimetype":"application/zip","path":"N2 on food L_2011_10_06__12_52_32___6___7.wcon.zip","platform_url":"https://zenodo.org/api/files/30a02c2f-5d73-4137-9d77-4f0b74ccdbef/N2%20on%20food%20L_2011_10_06__12_52_32___6___7.wcon.zip","sha1":"26f611fba592db60770316d80132c20608fbd2dc","sha256":"aa93faf81ac7f918ab53025278574976181a262fda3126b905be07c00e82f033","size":19619923,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1003567","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1003566","ext_ids":{"core":"144725395","doi":"10.5281/zenodo.1003566"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"hgp2qpa63zfelcc4wmk4otlxxq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1003566","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200216111242","terminal_sha1hex":"371b40ec2add773f7ab7814ae3fa72f5193a36fe","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1003567"},"total_size":303795909} +{"archiveorg_item_name":"tandf.figshare.com-8798045-v1","cdx":{"datetime":"20200228221140","mimetype":"text/html","sha1b32":"3UDJSDMUR7MOACLKUJLJFASST4IKDCYR","sha1hex":"dd06990d948fd8e0096aa2569282529f10a18b11","status_code":200,"surt":"com,figshare,tandf)/articles/enzyme_replacement_therapy_with_recombinant_pro-ctsd_cathepsin_d_corrects_defective_proteolysis_and_autophagy_in_neuronal_ceroid_lipofuscinosis/8798045/1","url":"https://tandf.figshare.com/articles/Enzyme_replacement_therapy_with_recombinant_pro-CTSD_cathepsin_D_corrects_defective_proteolysis_and_autophagy_in_neuronal_ceroid_lipofuscinosis/8798045/1","warc_csize":9727,"warc_offset":62948806,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200228210826694-31539-31602-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200228220952095-31588-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":12,"file_meta":{"md5hex":"75bd821e239fa94b6c3f089d63a9dec1","mimetype":"text/html","sha1hex":"dd06990d948fd8e0096aa2569282529f10a18b11","sha256hex":"10644418948b44ef39d68544622ea6c21b3094d365f3d96bdd45e4ad0a390ede","size_bytes":65954},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.8798045.v1","https://tandf.figshare.com/articles/Enzyme_replacement_therapy_with_recombinant_pro-CTSD_cathepsin_D_corrects_defective_proteolysis_and_autophagy_in_neuronal_ceroid_lipofuscinosis/8798045/1"],"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"3aad3a07b2a97c3179dec6de5f64c706","mimetype":"video/mp4","path":"kaup_a_1637200_sm4513.mp4","platform_url":"https://ndownloader.figshare.com/files/16125503","sha1":"4a6439b0bcd2c30546ee7a0a60dd4eeaabf4126f","sha256":"48a9ea2bd7937985217562f1ae07292437ce04b711edcd8efb01741525092d8c","size":13621123,"status":"success"},{"md5":"9bf21a499a831fb79e45b504a1293405","mimetype":"video/mp4","path":"kaup_a_1637200_sm4493.mp4","platform_url":"https://ndownloader.figshare.com/files/16125506","sha1":"9e0efbaaea25518aa4e9941f4d377eb2db6dbf86","sha256":"77ef158df2bf9453e4484d1d5efb8602d9025e2a3d849af5db70e98fa6d54a37","size":92269411,"status":"success"},{"md5":"d317675796a3028932c0169d32fe0916","mimetype":"video/mp4","path":"kaup_a_1637200_sm4463.mp4","platform_url":"https://ndownloader.figshare.com/files/16125509","sha1":"286ee96fa7edd57ad39176db9670298dc2e66698","sha256":"a2b476ebfdb4c18cda86c68fc640456d295e11fc98b5d846bfd36c0ee4a4e7bc","size":38162250,"status":"success"},{"md5":"302dc9fac62da016fd46b37828ef09dc","mimetype":"application/pdf","path":"Supplemental Figure 1_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125512","sha1":"59d79837bf9467959ba7a2bc35a0e595fd26a4e7","sha256":"94165fa67cf1b51d915609a03e11713e6b3f4554c61dec11be65216c38fbc8a9","size":1511687,"status":"success"},{"md5":"6f6029eda17bea63c4ea0c96671345a9","mimetype":"application/pdf","path":"Supplemental Figure 2_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125515","sha1":"845e41b018ff74b4b0b841be919eb552a3d75263","sha256":"426a2c6cdd7d2c599caaceed8ccde00965917257c40a04981799165199321dc7","size":2399827,"status":"success"},{"md5":"26f76c126db8bda86d608b6325757cb2","mimetype":"application/pdf","path":"Supplemental Figure 3_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125518","sha1":"b53b9ef82ed9d16f8c54778cba9eea3b9461e968","sha256":"291dc47ea0b7950e251fb1eb196c39b273ea440dbd4346f75c434e160d6c318b","size":16674330,"status":"success"},{"md5":"6a783fd3d8d43351a64d60fdab9012dc","mimetype":"application/pdf","path":"Supplemental Figure 4_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125521","sha1":"3d9010307726543ec76d8881e94f54b58df1cbfc","sha256":"714dd336289e10743ce82a599f2154266bac8fde3d324a30d62c931343148723","size":1146536,"status":"success"},{"md5":"03f7362eb084130fc3dc9e1b2467aab7","mimetype":"image/x-eps","path":"Supplemental Figure 5_Rebuttal.eps","platform_url":"https://ndownloader.figshare.com/files/16125524","sha1":"badc498455aa9c6cfa64872df3207c1a8745d76c","sha256":"88c26e634c85827fbf911459337594aa93892f0ab4d5a278fbf2ed6e4ecff3cd","size":26439002,"status":"success"},{"md5":"b391bcc2350ef5ac173346eb2b8940c4","mimetype":"application/pdf","path":"Supplemental Figure 6_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125527","sha1":"77db6f541cfa0fa823e6a15f018973ddfdf1f18e","sha256":"ab28e533e5d1fb4aa99285d8bcb0403edefa667adf9a45adb6d467be8f0fae7b","size":22433732,"status":"success"},{"md5":"b774f1f341be0e1e5df03909340928f4","mimetype":"application/pdf","path":"Supplemental Figure 7_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125530","sha1":"1e32b7c0904618e919cebdbf07c0d4448bd9ad83","sha256":"332bdb6a56a0d5b0af73c2ad526b7e4364e56c1429993f465921544fc8e23b41","size":14853073,"status":"success"},{"md5":"e7ea0f81a3dc676a13768ce3a2e087db","mimetype":"application/pdf","path":"Supplemental Figure 8_Rebuttal.ai","platform_url":"https://ndownloader.figshare.com/files/16125533","sha1":"5e52fb12d482624b3a34f702ad7145aa91775f1f","sha256":"73ca74589b81bd13bbd93c27654a9f82ddb60f2e3e4de8a785ee553c32c18052","size":3188157,"status":"success"},{"md5":"eaf9f944b5887f94aac927a969a57c4e","mimetype":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","path":"Supplemental Figure Legends R3.docx","platform_url":"https://ndownloader.figshare.com/files/16125536","sha1":"a004527c516c52a93fc4fe5b630dbfbbd22bc7f4","sha256":"f12e9445602df8ee77dd55d9783c21c5273d1ad027a6c7632681850a9b9634cf","size":28707,"status":"success"}],"platform_base_url":null,"platform_domain":"tandf.figshare.com","platform_id":"8798045","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.8798045.v1","ext_ids":{"doi":"10.6084/m9.figshare.8798045.v1"},"fatcat":{"release_ident":"aaaaaaaaaaaaarceaaaaaaaaai","work_ident":"aih4hrrgwfdajh7oyfdfbuejm4"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.8798045.v1","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200228221140","terminal_sha1hex":"dd06990d948fd8e0096aa2569282529f10a18b11","terminal_status_code":200,"terminal_url":"https://tandf.figshare.com/articles/Enzyme_replacement_therapy_with_recombinant_pro-CTSD_cathepsin_D_corrects_defective_proteolysis_and_autophagy_in_neuronal_ceroid_lipofuscinosis/8798045/1"},"total_size":232727835} +{"archiveorg_item_name":"data.goettingen-research-online.de-VCNHOR-v1.0","cdx":{"datetime":"20210828183217","mimetype":"text/html","sha1b32":"JLA4RLBMATAMR5HBRY7A7MYRDACQASAO","sha1hex":"4ac1c8ac2c04c0c8f4e18e3e0fb311180500480e","status_code":200,"surt":"de,goettingen-research-online,data)/dataset.xhtml?persistentid=doi:10.25625/vcnhor","url":"https://data.goettingen-research-online.de/dataset.xhtml?persistentId=doi:10.25625/VCNHOR","warc_csize":24270,"warc_offset":2864981731,"warc_path":"archiveteam_urls_20210828183934_ef2ec1cf/urls_20210828183934_ef2ec1cf.1606352862.megawarc.warc.zst"},"file_count":10,"file_meta":{"md5hex":"b311b8ab41b70630a4c7253947273505","mimetype":"application/xhtml+xml","sha1hex":"4ac1c8ac2c04c0c8f4e18e3e0fb311180500480e","sha256hex":"1210d006b4f8744860878b3e62a188882c0f6edaada7766236e31466e968550a","size_bytes":231346},"hit":true,"hops":["https://doi.org/10.25625/vcnhor","https://data.goettingen-research-online.de/dataset.xhtml?persistentId=doi:10.25625/VCNHOR"],"html_biblio":{"abstract":"This data set contains the stellar velocities used in the paper. It also includes profiles and parameter distributions derived using CJAM. It does not include the raw MUSE data which is available in the ESO archive.","container_name":"Göttingen Research Online / Data","contrib_names":["Göttgens, Fabian","Kamann, Sebastian"],"doi":"10.25625/VCNHOR","publisher":"Göttingen Research Online / Data","release_date":"2021-08-25","title":"Replication Data for: Central kinematics of the Galactic globular cluster M80"},"ingest_strategy":"archiveorg-fileset","manifest":[{"extra":{"description":"The MCMC parameter samples used in the paper. Each column corresponds to a parameter, each row is a sample. See paper for units.","version":3},"md5":"4b3b3a6ad215a7757d456c172ff56d9b","mimetype":"application/csv","path":"cjam_samples.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/F2LU0U&format=original","sha1":"109ae626f645ef8aaeba314fda1f911e63337926","sha256":"05bce65f0612c2a222c91a2f15f6d44abebf73ea6fd868f50475de099aebe84f","size":89857910,"status":"success"},{"extra":{"description":"The radially binned dispersion profile, computed around the Goldsbury et al (2010) centre.","version":3},"md5":"dc1e42d9421a4bb5b08510823696f1df","mimetype":"application/csv","path":"dispersion_profile_binned.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/LHMOTD&format=original","sha1":"1c9a1a4a66cdc38d2b692801a2a5248401eee163","sha256":"254551bfac4e0d804c317157f43f17fb6fa4cbc213cad2bcddbee08b92bbb464","size":340,"status":"success"},{"extra":{"description":"The dispersion profile from the CJAM Jeans model for those models around the northern centre."},"md5":"5b1cb03ef609951767310175a0f7d967","mimetype":"text/csv","path":"dispersion_profile_cjam_north.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/APSJ93","sha1":"400e11f369727475e97a0e66b9e409f2008d9083","sha256":"571e98e6ec8f40162a57d5e877e2b4875357d03ab5ac3b43f9e7765556ca7328","size":42442,"status":"success"},{"extra":{"description":"The dispersion profile from the CJAM Jeans model for those models around the southern centre."},"md5":"92bbda1e71a2118fa1a535741bc46198","mimetype":"text/csv","path":"dispersion_profile_cjam_south.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/TNXCTN","sha1":"822a2ddb20faf011d5b41dc13a0b8e5118eabf9d","sha256":"95642aae9ded6dae4dbf78694008a8d58f8ee4a56a640287d25bd39072277e6e","size":42426,"status":"success"},{"extra":{"description":"The dispersion profile from the CJAM Jeans model for all models, regardless of the centre.","version":3},"md5":"8ae667c8d16c867330fe716bda02a12d","mimetype":"application/csv","path":"dispersion_profile_cjam.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/MQQ0WI&format=original","sha1":"3fb0cb87069b9b846ee9a1fcf3badd8402126cc6","sha256":"fdc1960bb31b52ae10e7cacea0f5831192ea236d0f8bbd58f47d79f1e4087b23","size":6011,"status":"success"},{"extra":{"description":"The radially binned dispersion profile, computed around the northern centre.","version":3},"md5":"41d35e140d0101799898affa5b4aafba","mimetype":"application/csv","path":"dispersion_profile_north_binned.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/LHGODX&format=original","sha1":"587dcc0685f9a87e90d232217fbd256a1c01f0a1","sha256":"aded9078fd38627cfd111b62d5bf090972637a1045843c1f8068979f145793e4","size":340,"status":"success"},{"extra":{"description":"The radially binned dispersion profile, computed around the southern centre.","version":3},"md5":"f2c6268fa1e32f63f5fc562ed5f39782","mimetype":"application/csv","path":"dispersion_profile_south_binned.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/SCDCPJ&format=original","sha1":"2c0d981aab039c10298ca4da4f97c97ca6760d4f","sha256":"ac823067259a89f430c9f20b036b33918e9919a79520366d327d38d4515a5649","size":339,"status":"success"},{"extra":{"description":"Mass-to-light rataio profile computed from CJAM models.","version":3},"md5":"54e2c9526d90faf5e98640edc813d7bd","mimetype":"application/csv","path":"mlr_profile.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/2V4AWO&format=original","sha1":"8d6c02b97a030c38f59c1137584d46b02143ea70","sha256":"17291b62fa52826d2aea70a9877be3afac4cd2b5bfa90f58d1b55ecda743b6e4","size":9609,"status":"success"},{"extra":{"description":"Radial velocities of stars derived from MUSE data. The columns 'x' and 'y' are the projected distances to the centre of Goldsbury et al. (2010) in arcmin, 'STAR V' and 'STAR V err' are the radial velocity and the associated uncertainty in kilometers per second, 'Membership' is the probability of being a cluster member, 'RA' and 'Decl' are the coordinates in degree from the HST-ACS catalogue of this cluster (Anderson et al. 2008) or from the unpublished catalogue of Dalessandro et al. (2018). Note that the external data from H. Baumgardt used in the paper is not included in this file. It can be found here: https://people.smp.uq.edu.au/HolgerBaumgardt/globular/","version":4},"md5":"d662751924d4efa918a7aa2f3d40f286","mimetype":"application/csv","path":"radial_velocties_muse.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/U1IQHD&format=original","sha1":"00e161c865b90c10e0d1ce7551a26a3aee8c3db2","sha256":"988fdc97fd7b9a570baf9061b3f67310b9e7cc37e6bde7dda4c2c3913a83aaf7","size":1082158,"status":"success"},{"extra":{"description":"Rotation components for the cluster core and outside.","version":3},"md5":"8a48eee9b698d44b55465d8db0d4c817","mimetype":"application/csv","path":"rotation_samples.csv","platform_url":"https://data.goettingen-research-online.de/api/access/datafile/:persistentId/?persistentId=doi:10.25625/VCNHOR/BKZABI&format=original","sha1":"9bf530aea2d58720ed05318f8f2ed86cce224823","sha256":"093ae34caa084633a09cda80dcfcc6f370548dd2d824c2e3eee2a67d316fea76","size":257362,"status":"success"}],"platform_base_url":null,"platform_domain":"data.goettingen-research-online.de","platform_id":"doi:10.25625/VCNHOR","platform_name":"dataverse","request":{"base_url":"https://doi.org/10.25625/vcnhor","ext_ids":{"doi":"10.25625/vcnhor"},"fatcat":{"release_ident":null,"work_ident":"wdpk5pwdi5fnxoug7ywssyuitq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.25625/vcnhor","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20210828183217","terminal_sha1hex":"4ac1c8ac2c04c0c8f4e18e3e0fb311180500480e","terminal_status_code":200,"terminal_url":"https://data.goettingen-research-online.de/dataset.xhtml?persistentId=doi:10.25625/VCNHOR"},"total_size":91298937} +{"archiveorg_item_name":"zenodo.org-3307711","cdx":{"datetime":"20200214062807","mimetype":"text/html","sha1b32":"FW22DQVP43I6LOFB5N3K7OE4KAAL2CFI","sha1hex":"2db5a1c2afe6d1e5b8a1eb76afb89c5000bd08a8","status_code":200,"surt":"org,zenodo)/record/3307711","url":"https://zenodo.org/record/3307711","warc_csize":10093,"warc_offset":7129013,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200214052831899-12129-12194-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200214062801643-12187-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":30,"file_meta":{"md5hex":"34b928a8e72862b8eb80afba4d9bcfc2","mimetype":"text/html","sha1hex":"2db5a1c2afe6d1e5b8a1eb76afb89c5000bd08a8","sha256hex":"88ec09e76d26072404ff5fec9d939429584f2b3de9a514701ea8c62173c776b1","size_bytes":59267},"hit":true,"hops":["https://doi.org/10.5281/zenodo.3307711","https://zenodo.org/record/3307711"],"html_biblio":{"container_name":"Zenodo","contrib_names":["SXS Collaboration"],"doi":"10.5281/zenodo.3307711","release_date":"2019-07-10","title":"Binary black-hole simulation SXS:BBH:1935"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"dc88bc5b4b33d89ff21f1ed0df5422a5","mimetype":"text/plain","path":"common-metadata.txt","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/common-metadata.txt","sha1":"38f3c6a7c7588086045292d2b7b53b8c391f4f08","sha256":"3865ac09ce311c64f215494be4182dab68f7004f6e2b4bb18ddb39d1cd297d64","size":98,"status":"success"},{"md5":"ba165a942c515ffbe03c236139982e3d","mimetype":"application/gzip","path":"EvID/ID_Files.tgz","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/EvID/ID_Files.tgz","sha1":"4de8b413d26131ac0c7969df554616e257f6922f","sha256":"a2cfeb97c647b12abcbc3a720ccf5fbd8a2c28ea002de0cce4c7e2f4365b1894","size":16444839,"status":"success"},{"md5":"7c6ddc969cc380e8a1bb37f6b896574e","mimetype":"text/plain","path":"EvID/ID_Params.perl","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/EvID/ID_Params.perl","sha1":"6965201aede0303461a31a62341a51ab9caccfbb","sha256":"fdbbff5cbd2bec93bf9565da295c0091a9fa49b5aad6bfa5772a3b5835cbfe1e","size":1574,"status":"success"},{"md5":"8c8016a38def3c0fcc26a576ba3d29cb","mimetype":"application/x-hdf","path":"Lev1/Horizons.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/Horizons.h5","sha1":"78ea880f91019ed3acce9217b8de3f8cac1ce132","sha256":"012ece3acafda48934e7cb96dda9c21c39927a38dabcb05b0d70f3571f57fec6","size":2210469,"status":"success"},{"md5":"56ad4962413e25b2157cbc054dbdcf35","mimetype":"text/plain","path":"Lev1/metadata.json","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/metadata.json","sha1":"d2ec33427b51c3af1c25da8e0d9392eaed032c2c","sha256":"6a8f2b0fb700a5e324d413cbde98a649cedbb10e44a485348b0c41f7feaef636","size":3827,"status":"success"},{"md5":"dbb02a1660a632b1cddc09dc95b6c94f","mimetype":"text/plain","path":"Lev1/metadata.txt","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/metadata.txt","sha1":"89368dc0106ffe80a17fa42b5568d37a9caa0505","sha256":"91f12a852439056b3f07745dfade60a65a7aa32b2163fd410b3280daefd0a82c","size":4833,"status":"success"},{"md5":"71a5da1bae38690b438b805c3fc769e9","mimetype":"application/x-hdf","path":"Lev1/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rh_FiniteRadii_CodeUnits.h5","sha1":"5ba7e99876ca85ff6adfac9d1fdea28709e9ed16","sha256":"077bc697c9b6f5863d77077620ebf1a1b0da760e0f54137a5f8820d078872154","size":463907883,"status":"success"},{"md5":"e7a41f188c9119ab2b89bf3e16b4773d","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"cf54f3f9869c74e1754325a5fe2dd3df41ff6bd1","sha256":"3611b9ee0b2a9ff409232d1d9ed15a6237439ce23aeca27f6bad3a85a62e1047","size":81863987,"status":"success"},{"md5":"28648aaf7da536fb1d7159ecb8aab4ef","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"703b18ea7007e80ef02ac05666fdef80b67afa1f","sha256":"f39a2034bf67ceb303caea7a58925efde23ef065b03b206ade9c7f88d79e0886","size":83126547,"status":"success"},{"md5":"1edc7b69b944aa22c922c36741981452","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"5f5fa26675c9afcaab23f3e89bb12265e763d264","sha256":"eb5b06fed8df1679fd5cfd6ca4b0278263ba2111a2347e56f3113b356c9503a5","size":81962220,"status":"success"},{"md5":"9fb33dc914b0b4bc8cc1dad2e432bfe1","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"43601934966c6fd243a7e59213bad483ba57328f","sha256":"7eab1a7c25e25dc8b8f4c8dba7a570b94ad77f325fb110270296a557aba68cbc","size":82990002,"status":"success"},{"md5":"f1a30567d6f746a64e00a594f6d7c73b","mimetype":"application/x-hdf","path":"Lev1/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev1/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"58f32838f5b8315060aa76e62a00108387093433","sha256":"cc18f53e162d41e460d49314aa1583e5543a72b045d8c756cc980ec43159a71f","size":462463734,"status":"success"},{"md5":"a4eb5734412bf0d3990ffbf4502ea3bc","mimetype":"application/x-hdf","path":"Lev2/Horizons.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/Horizons.h5","sha1":"6c604def9f795b0fc3b191d852d1571fa171e866","sha256":"590eee07d0f083fbe99cd57f8a1c46b0654a08777ce5c99867808aa2d999a8b1","size":2184031,"status":"success"},{"md5":"73847cfd26d6111f91f79b15618e4397","mimetype":"text/plain","path":"Lev2/metadata.json","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/metadata.json","sha1":"73a67c9db48e93c99fd0de0ccaa04f1f434870b5","sha256":"7dd88992ebe230f5cefc3a5981046cef08821a74b23d7123ac1783bb8d0a906d","size":3828,"status":"success"},{"md5":"22945ea9aafa567cec79e3a3f0dcc9e7","mimetype":"text/plain","path":"Lev2/metadata.txt","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/metadata.txt","sha1":"debed037ec06c76206a76d2603f1e6b37e101392","sha256":"f875e400190e7851fbc5b475b823850c2cbcfd8e78d9b8534d2a59066e51ca3f","size":4835,"status":"success"},{"md5":"9b5520ec0a5b6e283cee3aa70bc955db","mimetype":"application/x-hdf","path":"Lev2/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rh_FiniteRadii_CodeUnits.h5","sha1":"8f4bfad6d053a3b322df60d1c2ac155936ede70a","sha256":"364db98c04ea6a598f953b312009b35d38583cc5b6457f0bfc3e626ce87eae10","size":461957707,"status":"success"},{"md5":"b7b9c93ccb1387d3c73e96ac55255a2c","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"78ce82bf532dd9cc93eb780337c2b3b5e19522ec","sha256":"ac0000f48386568819215cb96429cd64a0f37331ad2eceaf1a461643157b6683","size":81407944,"status":"success"},{"md5":"090ccd8dca7cc0979bb99bdd349b7917","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"80c597154cbc5d798c220fccad395d87a532ff19","sha256":"8a552d82e9eeb4baec0b19825939d1acd33a1aa941b4c6819f711fc2ea0601cf","size":83078110,"status":"success"},{"md5":"d66e0bb4d7d5a4b4ad58f45dea94e552","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"ac9fc656921552366f7f029eacf0856d8f8ccfa2","sha256":"76174931d71fea4e5945e8f9c76bdd9a5b4706e1cd293364915a13ade65fcb62","size":81769730,"status":"success"},{"md5":"61d4516c85cc7f1f749ea852f28bd121","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"1f0445f81991f9dd9efef38a8ebe4d7291674de2","sha256":"6a25c9acd8f6e5db5a0b02ec6305e94aa7e5d04afaed20d4fc5d5b9e2d451480","size":82850372,"status":"success"},{"md5":"5f5a522dafaba58fa65fa1955f8c92d7","mimetype":"application/x-hdf","path":"Lev2/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev2/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"9aceecec74afe68ee4db0e508732ec41d2951b49","sha256":"46c9bbd6324db13dd267f6c9bea1afbc291f74e4dce1b986fcb88d5b673df495","size":461925763,"status":"success"},{"md5":"6895e71861b580d8c7cab40d8f228c95","mimetype":"application/x-hdf","path":"Lev3/Horizons.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/Horizons.h5","sha1":"7d15a8b16624127dcc436fe7c9c5f4e2ad399701","sha256":"9f4264cf1c66869762f4b5906d0149cb1ce7be34a7347f1411bb6b5d5f149d17","size":2162410,"status":"success"},{"md5":"551466d8e5f7fce29f700ba7c9466611","mimetype":"text/plain","path":"Lev3/metadata.json","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/metadata.json","sha1":"22a718928b46de84e40f0729ed930b9ff493ec94","sha256":"5c8de4273fa154d817e17e85aa2dc5d4e69bfe1796a61fb53166535bdd631a2d","size":3821,"status":"success"},{"md5":"be92a00cc9d7eb5a2b0889cbd7354387","mimetype":"text/plain","path":"Lev3/metadata.txt","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/metadata.txt","sha1":"c51b3910039654c7b1b7c6d2d96dd5e500661b76","sha256":"a5eb27ed8832f77ff3ce7293187daa9491ebeea1b52462bd93ef8fc4d5536c2c","size":4829,"status":"success"},{"md5":"0f4ad0447fc826bca5dbb6bb15159db2","mimetype":"application/x-hdf","path":"Lev3/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rh_FiniteRadii_CodeUnits.h5","sha1":"f78e350db595d3b7f8d9dee631cf63ae22d10eda","sha256":"1162fdad3890d607e71c76abb91cc1cfc993d475e222b8ded181943875a79965","size":460713089,"status":"success"},{"md5":"890a13b6d77886431b0e8abf6e735fd7","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"96c33af7a0e830cc263a51cc136bce5179c64047","sha256":"f6874cee5596b8e287a9ad11b23c5a2d0d696cab69c7b86c927066873b6ae1e5","size":81157429,"status":"success"},{"md5":"0a1ca0255520ed56dbd4445d7f16603d","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"9cf3d2d889608102b30f4fa43a00450dff2a7908","sha256":"1890d8f1a5442c964331d9286554fa53ca41cf3b9d1f789ee22c2882581988c3","size":82848632,"status":"success"},{"md5":"12a4d523c3af2e16515188a1d0eaa09b","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"38302f4b0a1dbb46bdf52f2c76459a9eff2066f1","sha256":"ba8d44b6663934df77c9825adc0990140a3c9488fa86df5dec626fe4e37400e0","size":81526389,"status":"success"},{"md5":"f27dbfa81258a06f40313cd62bb158fe","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"5c685125cc8ed551469a336c68a0d63f08b940d0","sha256":"52561c90df1f98b284b947ea00b604bc6d3fdd52094cc1ff9c9a240121afe622","size":82701578,"status":"success"},{"md5":"8620556c8ef9dd95308bfc7597e431c4","mimetype":"application/x-hdf","path":"Lev3/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/1f94e426-db6f-411a-9bd8-7ca5688b2465/Lev3/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"fee256a32792b4dc737918ab1a62408d6ad3e9c4","sha256":"d18c129a2139d03c8a84ed301155fd99a6b422ef109d50e689855955cf8c1f63","size":461471187,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"3307711","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.3307711","ext_ids":{"doi":"10.5281/zenodo.3307711"},"fatcat":{"release_ident":null,"work_ident":"kq4rypbczje7vjkgauc5olclaq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.3307711","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200214062807","terminal_sha1hex":"2db5a1c2afe6d1e5b8a1eb76afb89c5000bd08a8","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/3307711"},"total_size":3782751697} +{"archiveorg_item_name":"zenodo.org-1016167","cdx":{"datetime":"20200301025736","mimetype":"text/html","sha1b32":"YIMWWTZ5EKW42KGDIGNWUPIDYX2BCOUR","sha1hex":"c2196b4f3d22adcd28c3419b6a3d03c5f4113a91","status_code":200,"surt":"org,zenodo)/record/1016167","url":"https://zenodo.org/record/1016167","warc_csize":9736,"warc_offset":6123903,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200301015118559-34077-34143-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200301025729427-34131-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"e5de432a4253e92a07b4f92d871e92df","mimetype":"text/html","sha1hex":"c2196b4f3d22adcd28c3419b6a3d03c5f4113a91","sha256hex":"88845f128fba290447abd79aa90f0f86e3fd7c851520bcf5f9e11383d93a2e10","size_bytes":40372},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1016166","https://zenodo.org/record/1016167"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Javer, Avelino","Currie, Michael","Hokanson, Jim","Lee, Chee Wai","Li, Kezhi","Yemini, Eviatar","Grundy, Laura J","Li, Chris","Ch'ng, Quee-Lim","Schafer, William R","Kerr, Rex","Brown, André EX"],"doi":"10.5281/zenodo.1016167","release_date":"2017-10-20","title":"JU345 C. elegans Wild Isolate (Merlet, France) | 2011-03-04T12:20:54+00:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"01367f81b840cd52f16f1da18408698c","mimetype":"application/x-hdf","path":"972 JU345 on food R_2011_03_04__12_20_54__5_features.hdf5","platform_url":"https://zenodo.org/api/files/b26cfb2c-168f-467e-922b-052175a68a7a/972%20JU345%20on%20food%20R_2011_03_04__12_20_54__5_features.hdf5","sha1":"5593c307872613ec5bc28f7d9807feddf580124b","sha256":"084e3cb98ab28048812b3184f0b42ff56f26e95de17b7d5740412a40077bcd09","size":21246213,"status":"success"},{"md5":"cd746244ff88cff778f92769205e4af8","mimetype":"application/x-hdf","path":"972 JU345 on food R_2011_03_04__12_20_54__5.hdf5","platform_url":"https://zenodo.org/api/files/b26cfb2c-168f-467e-922b-052175a68a7a/972%20JU345%20on%20food%20R_2011_03_04__12_20_54__5.hdf5","sha1":"a9a47cf4c1ac59eb3ed63a3848a4db4b10906b2a","sha256":"708416dbd095afc9585fbe5e4353c49592a376ee7a0dec8a3434cff70bc1dbca","size":330770328,"status":"success"},{"md5":"6451189679d200dbce809af2f35b82ca","mimetype":"application/zip","path":"972 JU345 on food R_2011_03_04__12_20_54__5.wcon.zip","platform_url":"https://zenodo.org/api/files/b26cfb2c-168f-467e-922b-052175a68a7a/972%20JU345%20on%20food%20R_2011_03_04__12_20_54__5.wcon.zip","sha1":"6d2f386f9fd68e26c38c6659021a354b90205b41","sha256":"d39ed71e56951b965159a4826bcfbaed742218a08bee6a49d502244235a886f8","size":19778829,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1016167","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1016166","ext_ids":{"core":"144856033","doi":"10.5281/zenodo.1016166"},"fatcat":{"release_ident":null,"work_ident":"uz4s7heup5fqthb72f6uqg4wgq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1016166","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200301025736","terminal_sha1hex":"c2196b4f3d22adcd28c3419b6a3d03c5f4113a91","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1016167"},"total_size":371795370} +{"archiveorg_item_name":"zenodo.org-1404410","cdx":{"datetime":"20200302075608","mimetype":"text/html","sha1b32":"6DB65FEVCS3V65RFAMZIAL25RJ6PNJV3","sha1hex":"f0c3ee949514b75f76250332802f5d8a7cf6a6bb","status_code":200,"surt":"org,zenodo)/record/1404410","url":"https://zenodo.org/record/1404410","warc_csize":12000,"warc_offset":40905523,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200302073516225-34453-34517-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200302075450003-34467-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":59,"file_meta":{"md5hex":"abfe0bcb207741b6e19c3016bb694297","mimetype":"text/html","sha1hex":"f0c3ee949514b75f76250332802f5d8a7cf6a6bb","sha256hex":"b0a8144ae1c3601ab41d61490794f9b1e0a2967acb3a685699e45c453642cd79","size_bytes":91739},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1404410","https://zenodo.org/record/1404410"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Family name, given names"],"doi":"10.5281/zenodo.1404410","release_date":"2018-06-19","title":"DWD European Weather"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"182f7fe0793cba6936c824e9eea6727b","mimetype":"application/zip","path":"20180628.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180628.zip","sha1":"2dfc1b743d92c6b53c5443bf827e0e66598b9db2","sha256":"5bd5b1859a89e65e6ca779aefce6bd5d7f26286ebd3d5358144163133e897d5b","size":124048454,"status":"success"},{"md5":"ca0281303e0d902739097333cbb2c760","mimetype":"application/zip","path":"20180630.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180630.zip","sha1":"a44dafa54960296979cbb0e9e91cec82ec42e8a2","sha256":"eaf2713536ceeb6862a17e8cd5d7ed0b8e3c5cf2b33cc65a8cafbbd47ddc23bf","size":125546856,"status":"success"},{"md5":"89df2ac188d10d9db5980ff98a47e63c","mimetype":"application/zip","path":"20180701.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180701.zip","sha1":"f53898b453ebbc233f24b157cc81caf3bb7f5326","sha256":"641ca0941c2b4c9d2536b63de3ef3c9c9d04869e28b50c54bf3b1ea14293c7f4","size":124452053,"status":"success"},{"md5":"69b4fd90b1ddae2f374ad8853a2d73b3","mimetype":"application/zip","path":"20180702.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180702.zip","sha1":"94143aa95d6b6e1cfdc5d96ce3481492e1b8e423","sha256":"86149ec2d531321ebe07e40df7271d9884b7ba5981cf2b76123c91da38d41e43","size":122935786,"status":"success"},{"md5":"7c1db8c87fdf64632e15b63d764a07d6","mimetype":"application/zip","path":"20180703.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180703.zip","sha1":"9fe03ca775416c2cf12a91dab67a21fb706ec73b","sha256":"3ba4bdcb4ea6e69e2534e19736b9c23a9546125c96cf96e913481867d06876e3","size":124944730,"status":"success"},{"md5":"bab41d1212db0209ae3a7730de4bd417","mimetype":"application/zip","path":"20180704.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180704.zip","sha1":"a1f151238a45d2ccbf9b8143e6a46cc91ba39ea0","sha256":"7e000bfebde797320fea5293873d20e4a248855be34bb68f826282aae913cb3a","size":125729238,"status":"success"},{"md5":"1fd53dfe9e4d28a57bfad62bd5f27654","mimetype":"application/zip","path":"20180705.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180705.zip","sha1":"995fbca7ee1c5c440f14a3b3ff18da408ba330dc","sha256":"2cf56b8a449da18d28530f4d965569b2eca5ec71d3fc20bd3d85f939e5415d8d","size":127000626,"status":"success"},{"md5":"386d134c6e826eaba8282bb109b4add9","mimetype":"application/zip","path":"20180706.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180706.zip","sha1":"4d143585daa634688d2d32a5d31f7ad41f3f6099","sha256":"e9aa15b6b6c0656a41db36348b8ef1954acd899ec5512023495774ab27e4a4d5","size":129418855,"status":"success"},{"md5":"9718e135a809a1207a73ef6134b96ed2","mimetype":"application/zip","path":"20180707.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180707.zip","sha1":"13cdaabc878bf6402fe6e0f13f5339c4792483de","sha256":"dc29956bfbbb2db87ddec9f56326d18846f04b58ab6692783d33dda1f7c454e2","size":129417196,"status":"success"},{"md5":"a8e0a155d952a374e70fdb27f9fb9633","mimetype":"application/zip","path":"20180708.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180708.zip","sha1":"c6d673b61d65f7b64e345b90f5e068fc3236e650","sha256":"d096afab366d03c9834852934d7de1e15112d6f034e8b4a3e45c81cc3a7fb672","size":126933738,"status":"success"},{"md5":"82319371b8f6f50e11f2658a228bd908","mimetype":"application/zip","path":"20180709.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180709.zip","sha1":"1f6cbdca0d824b990daf4304746d044b9256f110","sha256":"ecfc97050b25248b0e22911bd4c8b20f74037d3dbdc7ee5664682197e3db7e92","size":125027685,"status":"success"},{"md5":"1a1d7e59261b460e8b02c03f3e99d331","mimetype":"application/zip","path":"20180710.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180710.zip","sha1":"2bd69f4c644ad3f98f589fb3ba16b35e56bc25ea","sha256":"022e166a5673b5b4e074b4d3ad597fb8a77d676b33784940cab2d9e904b163f0","size":125476081,"status":"success"},{"md5":"905cc39ae50668c1194caadb6f7d0a2f","mimetype":"application/zip","path":"20180711.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180711.zip","sha1":"20fd8fbf5d95fbe71b96b40be6b1088fcdf32243","sha256":"a95d59de057dfd6073a89bdf5a1a4103560a129b53626b668ed53e096bb1527d","size":127229905,"status":"success"},{"md5":"0159941f0b468eac257981404e1f496b","mimetype":"application/zip","path":"20180712.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180712.zip","sha1":"e4a78954b375d6349a2b4f220964cf7f4a2957b5","sha256":"d79923b39c916153f6d21cda7d7dc8206b23d5cc1d2858afd0f5357e39f598eb","size":127513662,"status":"success"},{"md5":"895138d9c25a7feb203ad64cbc1cf3ab","mimetype":"application/zip","path":"20180713.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180713.zip","sha1":"c9001a611eaada015dfb30e1c021991793b6aa26","sha256":"6dcd33a5e2b5a23671c3087d9deedf443b2fe54ae5fc5523b00df0f9db88103e","size":126892581,"status":"success"},{"md5":"c092ce004a001136ae2f00e508e453ca","mimetype":"application/zip","path":"20180714.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180714.zip","sha1":"3152e73f9e4af438e243cc568b494342b36fbf51","sha256":"467a6e74d69e30a84f9ac11441ed199b4e77cdabc176f53692f1b9dfb269fd75","size":126102971,"status":"success"},{"md5":"1a88aeb01c1fe926059f1d456bfc279d","mimetype":"application/zip","path":"20180715.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180715.zip","sha1":"7f0f7d9bce3e34089d29d31c0213c3d8c14db625","sha256":"9c6c723ee7b3fb15b8e4adf31bd8c80e1ed6198e665f453e0bd4c6935410de69","size":126338316,"status":"success"},{"md5":"5c5279c4a7cf8bd51353a9ba6a5ef1db","mimetype":"application/zip","path":"20180716.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180716.zip","sha1":"a4b20969170fa69c5bdff22331c6d98c8707cb4b","sha256":"5a01419c47fd94a38e3422d1b98c99e368ced5211335b44c1f818497dd5ede79","size":127836100,"status":"success"},{"md5":"3ca8c4054e1b48c7a7f9878100941355","mimetype":"application/zip","path":"20180717.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180717.zip","sha1":"6457ec1981449f746ff94fb1e606b317607e0954","sha256":"548ae48b8ef0cdad9c7701ee01f32b4f0eb08dfe87359a090391ee9673b07a8e","size":126789734,"status":"success"},{"md5":"a922e2e416a618a111a3edd5c32ce8e1","mimetype":"application/zip","path":"20180718.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180718.zip","sha1":"75a78f602df8693566e19381a1377787f617caf6","sha256":"4e6b3fe31666d03b536cff3e0a303ff959fe7e484378161f711a37d139c7b832","size":127505035,"status":"success"},{"md5":"2abf0f9beb3d0430fdf0d7b22a23530d","mimetype":"application/zip","path":"20180719.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180719.zip","sha1":"82ccd32d35fc234eb83f4a539525835338918547","sha256":"f8af80c295fc876d550fd9d7b26e31c7e049770acaea57f20b577effe3705142","size":129419652,"status":"success"},{"md5":"d15105bfa2fd4b49f01b4817fb1bde5e","mimetype":"application/zip","path":"20180720.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180720.zip","sha1":"537ade29a1a2d48d1dc65e691360c27de14e6259","sha256":"7388622224beb7526b5d52052c372ccba09d5311cc176b831fd387b115ba5dae","size":128969929,"status":"success"},{"md5":"9c062387ab4ae2a8d36fdcad3f7f5d65","mimetype":"application/zip","path":"20180721.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180721.zip","sha1":"b5e1a6f1808727f9efac834efa8a7d22fbccf61c","sha256":"d07f2022f695c5c5d2143eb3f4aeae796b8b0fe738e6e4d1251e33f923651e1b","size":129300706,"status":"success"},{"md5":"df777328eddd99b45f5417750147f1a3","mimetype":"application/zip","path":"20180722.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180722.zip","sha1":"dade1894fe19bda2501319e5e8f791c1dec4e554","sha256":"49cc91f76c2e3f5b185173f7c29d38f10a5bd5574f0ddcbca5abe80a860a2f40","size":129215962,"status":"success"},{"md5":"d4b4ed4ec5947c73298da3173dc1fc38","mimetype":"application/zip","path":"20180723.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180723.zip","sha1":"f9078cfb3e476f8a9d97b14d98220ff20a8cdd75","sha256":"9b00b4e74182a230dfe15740ae660dd850c3abe5c95ff8eeb8f8e6f5d97e9cc8","size":128617862,"status":"success"},{"md5":"2d4cfef62f4c0add8094409318ea4e70","mimetype":"application/zip","path":"20180724.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180724.zip","sha1":"fec687acd5bc882d703a0b41b938b216fe6ef467","sha256":"69f2901f01055a085d3166ec91e88694d0fc5cedb570638a769dc59aca72e49e","size":130273727,"status":"success"},{"md5":"6d8996c983b668af928f67f22c410508","mimetype":"application/zip","path":"20180725.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180725.zip","sha1":"70ec84f047f455cc54929c6bc2d9b6a134d93adc","sha256":"0926c14e644259a56e2879227a1cde4ef8d26eb539a4867ce9989d5438dfc45f","size":130072041,"status":"success"},{"md5":"53a8b1252db23a56811a0bf1495d6a67","mimetype":"application/zip","path":"20180726.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180726.zip","sha1":"ee0204781b5f9dd2cfe22975d52dc62ce9769111","sha256":"0b392b7f729bf6834c4719e9827ee114ddb813434293711ac05e999a739b76a4","size":129953121,"status":"success"},{"md5":"7ab4c566b97e1c5d3e8545e8d5af87f9","mimetype":"application/zip","path":"20180727.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180727.zip","sha1":"a1348d064aece797cb4566af6cf112f270a2260f","sha256":"6f1725b1d3e2585000c1cca0ee51a0354994ac7f62c566ed507c5721ec9336ea","size":129462137,"status":"success"},{"md5":"d3d051a6ea03fd64839b092d6a9a6dc3","mimetype":"application/zip","path":"20180728.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180728.zip","sha1":"98edfcfbc6c66f13f77c4358cc87fd609b3fa169","sha256":"cdf077e0d9dd8056db867e6f1b332c464314fe2d16ffd6610d8b928ac937c232","size":127227897,"status":"success"},{"md5":"a18dbb7c1d19e1fd2d5eaf34110e8248","mimetype":"application/zip","path":"20180729.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180729.zip","sha1":"1bfe455fd3b7252a2631c211580a362e56408f5c","sha256":"4ec3d0774ac8233a15dbd603d1e232c2767b405e37bfb4da448ea929ed4ccb81","size":125273483,"status":"success"},{"md5":"ed2e9764c5c5f5b509a88a662cb1bf19","mimetype":"application/zip","path":"20180730.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180730.zip","sha1":"a6568e6ab7165ae5d6c5b645759ec63f7b98b506","sha256":"aa8d674d4010e376946e5329039e8a7216defb1d4c9ba1bff806c150f4ae8298","size":128505961,"status":"success"},{"md5":"2a03e24fe9b68f3c4edb7e12f534c878","mimetype":"application/zip","path":"20180731.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180731.zip","sha1":"d855f1712d242d0e7eb5057bb536393c0603dd28","sha256":"522ac74bc610b043930cc401ff133019224d0cd096513df21c544b8f277e7fed","size":128194085,"status":"success"},{"md5":"980ddf9f7decd45c652554dfa7795745","mimetype":"application/zip","path":"20180801.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180801.zip","sha1":"a502bded5c074f4d72f2e12bb35f4e38a547b739","sha256":"9d313b09a16da581937403b6e8928536180bd4b106866ed67d7c7e4ffb693fdc","size":128479748,"status":"success"},{"md5":"e35bc55e0f510e90d09180d9668c65ae","mimetype":"application/zip","path":"20180802.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180802.zip","sha1":"b77653c533352b382999255f2490a8211c9a8f8e","sha256":"ef98c77794a57f94532d0db832e0e82f2f84c93249a4714b8831212748781466","size":127540393,"status":"success"},{"md5":"d3c87c76e21c11dda012f241abe7952e","mimetype":"application/zip","path":"20180803.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180803.zip","sha1":"ca9c4a19133418d57d2b3b23d8570eab683125aa","sha256":"3a0cec72d3e96903955678a35f3cead6b03f4dc98e81c63fbc905a18c317224e","size":127906233,"status":"success"},{"md5":"61ae662b0be8ca1db8e2fb1bfba1c337","mimetype":"application/zip","path":"20180804.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180804.zip","sha1":"cf0eda7beda90eae13b0c079ee4326f9cdc2c321","sha256":"45f4b1289ff172bc884ad35a489e6af147551525130f0f61e2b6f9eecd137281","size":128858996,"status":"success"},{"md5":"faf0e559822e3ff6793dd921bc4ef207","mimetype":"application/zip","path":"20180805.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180805.zip","sha1":"261bc0ae24276d5f8f8c329b267fd30a8d4519b6","sha256":"610fb4f0763952da583d2061a84acf472cc847b745b60f5e2903daac05d67d9d","size":127539214,"status":"success"},{"md5":"6f420fc8196107a998a41bad1df23e51","mimetype":"application/zip","path":"20180806.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180806.zip","sha1":"bd72fb9d69ac861524eb903547e9bb122022faae","sha256":"2ec84430c70f2046d420aefdcc7ea5da5833afa1fe8b0bf86934a28e52bde8e5","size":127178686,"status":"success"},{"md5":"69a83d3c5795d82bd2480a0cd11989a9","mimetype":"application/zip","path":"20180807.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180807.zip","sha1":"0d1c113108405355d8a1619dbb605edd97e08390","sha256":"4a48b6448cab4327e204ec6f4cd88a037155e5ddcacc15ef0c3d3d1a171d2514","size":130274784,"status":"success"},{"md5":"18010b1dbb259baaaaa1d1b34a8b9e78","mimetype":"application/zip","path":"20180808.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180808.zip","sha1":"29f8c4449324cead4143235dec8e2f9490a7acda","sha256":"3b9e3da05209c62baa59bc215f3e09b94ed5f6c67bf3508d47af5a6c9be47270","size":131116209,"status":"success"},{"md5":"dc9c7f536dd67cbe751612080e007aea","mimetype":"application/zip","path":"20180809.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180809.zip","sha1":"83784b85c99396188f151184774da107d3d37b7c","sha256":"5458414ed19d5342c4298f77466b7df31c7561f3cce94d9e532b623352b9ad3b","size":129721875,"status":"success"},{"md5":"e1fba16ee6b5cb1a64a02e5f1be7ee4b","mimetype":"application/zip","path":"20180810.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180810.zip","sha1":"aa08e1f9b9d180310b5009d61678e01441743126","sha256":"0ccf24e8f4ff8258726dc8b37811190e10fb22bd2b14474f8cddb14d7267cc0c","size":128450516,"status":"success"},{"md5":"5c5be9b1f0baa493ea3b267410db1b1c","mimetype":"application/zip","path":"20180811.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180811.zip","sha1":"93aeea1caedf2ea2d1c15992cec7c49d03d300f9","sha256":"d7197204620f63a16d67d51f1f539e0a28257597e8eee6c57a056dfca710c85c","size":127393237,"status":"success"},{"md5":"75e2dc4c5593d4d213715b96af7db690","mimetype":"application/zip","path":"20180812.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180812.zip","sha1":"7ce8f1179cd2833c615350fbac0676c74d2171e0","sha256":"68237f7d7654394bd3c82f9c68126daabdefef631f1e229538314a0cad4d503e","size":127216972,"status":"success"},{"md5":"fe983fd25bdbf1becab3b232b9fb724d","mimetype":"application/zip","path":"20180813.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180813.zip","sha1":"91c7f9ca9c68cd728f78a4af0954797f802c0add","sha256":"a695868fc37b90a9c7b72ae3a72ffcf0470896681b69c5468af0d7a0cdbb7e6a","size":126201407,"status":"success"},{"md5":"8ab510659bff1d0e612a609a39f1fa48","mimetype":"application/zip","path":"20180814.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180814.zip","sha1":"008fb00d717b2b72a29bc71ee245afe279da651d","sha256":"c566555a637c212d223e8d993dd7ae2c87ff0d12acc13eb7b7a0bd2041408469","size":126841984,"status":"success"},{"md5":"fe5e68f1c2a338a29dac8fd0153c5e4c","mimetype":"application/zip","path":"20180815.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180815.zip","sha1":"1145a58c15e5e0f7b999a1fb6365ae14519c988d","sha256":"786415b34c9821e004da51911785f9afde0c728274add2f3ca0f55df042f72d3","size":128603354,"status":"success"},{"md5":"fada06dbfc5ee9acb0da7241e6745541","mimetype":"application/zip","path":"20180816.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180816.zip","sha1":"015caea50e8164fb38290e7913a6d38af6c9a1d6","sha256":"7d145b08736208027a6ce6904819ce253ea0435c17fd9e3268e1df06e84f7183","size":130989096,"status":"success"},{"md5":"090b2c5863784e783971646364a025a6","mimetype":"application/zip","path":"20180819.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180819.zip","sha1":"f9e00ef0cc14636f213f612989f347f4a697bf09","sha256":"08b844ab56425eaaa994315979b79cab4df76a240c0b230649bbcf277dfd90f1","size":125168381,"status":"success"},{"md5":"36edcffde1f8ee3a7146d5e78d8195c2","mimetype":"application/zip","path":"20180820.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180820.zip","sha1":"72fe599fa612a1b3f7bce7613040dec427647352","sha256":"d9636bc337af061f9a92ff2c075526b88d8a3c87e5b5322b2d859733e1a81f63","size":127843797,"status":"success"},{"md5":"727c5f6ea7e24f205e113d3867bc323e","mimetype":"application/zip","path":"20180821.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180821.zip","sha1":"ecf9149f869de5b15e33a777747f577f190a23d7","sha256":"e683ddc4a1c51c9f0d8fa70da2f16bec7bd61ca21c89c5326b201cf9405395ca","size":125998744,"status":"success"},{"md5":"17377d2732bd7d00f07c192b06335c6e","mimetype":"application/zip","path":"20180822.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180822.zip","sha1":"76a7ab5e77b2ad9bba7a217a66d1aff6141357d9","sha256":"190bdbf179d28a3e28f71e8592a73cf741355c746f1e39862279d80817a73952","size":125640622,"status":"success"},{"md5":"12ca57fcc97d2a5d1ea9a75d3be90a08","mimetype":"application/zip","path":"20180823.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180823.zip","sha1":"4b64fbbb7c7d3f52cb07be01aea2acb622d189d5","sha256":"0db8804ff9f60dcdacdf93653972c464f79a81f04629342765c4d206dc051b29","size":127778150,"status":"success"},{"md5":"75983a0bf60e1259bb76422488a7c77e","mimetype":"application/zip","path":"20180824.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180824.zip","sha1":"21a04c35f4bf11e251746d3aacf66c1a2285e7e8","sha256":"c9ecb7dbbc63e3456bdf37c2de06dcdc1789b898a1e4b191fa567223d39cd2e2","size":128677596,"status":"success"},{"md5":"b4fa11e31fb046de2d770ff7bd86f93b","mimetype":"application/zip","path":"20180825.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180825.zip","sha1":"7e6bd88895f99002e51a70485eb3b85807660202","sha256":"70f21be806a62e977b0b4b9884f1465c6b2e5f214b441d8d137bcce160f5c3e2","size":129475592,"status":"success"},{"md5":"6a952443debe9c6df9344128e0bf5626","mimetype":"application/zip","path":"20180826.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180826.zip","sha1":"f5df962ca85a6ac8fe126c8d4b28d436ae19e57e","sha256":"e7c6e59378ad6137c966d17d82143dbdbbcb8309376f62b8316c8aff53694b12","size":128123340,"status":"success"},{"md5":"56d5bc0229ebaa40b186251841a52a32","mimetype":"application/zip","path":"20180827.zip","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/20180827.zip","sha1":"c054180f874f70849a176e9eaaa35e60bb2405d8","sha256":"21ee13d2d03d6a31c9043b0829137ef98300cb4ef068ff8e65ae5dcf4967b53e","size":127642437,"status":"success"},{"md5":"95692029fefc00e6f1cc50410d3e5cd9","mimetype":"application/vnd.ms-excel","path":"opendata_content_de_en_xls.xls","platform_url":"https://zenodo.org/api/files/670f49be-383c-4231-a438-c5dea581fdaf/opendata_content_de_en_xls.xls","sha1":"3c5b96b7f9f170ae8e83363b2f374ff17b1f217b","sha256":"18a98657721b6bff6bfcda5302bdfcbeae79efeea46d6505647e5a1b6a46019c","size":360448,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1404410","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1404410","ext_ids":{"doi":"10.5281/zenodo.1404410"},"fatcat":{"release_ident":null,"work_ident":"ywdnioaipffkthcr6mdkfpcbdq"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1404410","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200302075608","terminal_sha1hex":"f0c3ee949514b75f76250332802f5d8a7cf6a6bb","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1404410"},"total_size":7402647976} +{"archiveorg_item_name":"figshare.com-9751934-v1","cdx":{"datetime":"20200229135146","mimetype":"text/html","sha1b32":"ZCJEDPGEBOQ36MLN5Z23B5HIU3QJRRCF","sha1hex":"c89241bcc40ba1bf316dee75b0f4e8a6e098c445","status_code":200,"surt":"com,figshare)/articles/current_and_projected_future_risks_of_freshwater_fish_invasions_in_china/9751934/1","url":"https://figshare.com/articles/Current_and_projected_future_risks_of_freshwater_fish_invasions_in_China/9751934/1","warc_csize":6804,"warc_offset":1373840,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200229125255118-33471-33537-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200229135144301-33518-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":2,"file_meta":{"md5hex":"06ab7b3cad9382b9dcfabeb0d1a9e12e","mimetype":"text/html","sha1hex":"c89241bcc40ba1bf316dee75b0f4e8a6e098c445","sha256hex":"29831dd686928805df37149bbdbee913ec77344bfe2b82f721d739a2bee4606c","size_bytes":31138},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.9751934.v1","https://figshare.com/articles/Current_and_projected_future_risks_of_freshwater_fish_invasions_in_China/9751934/1"],"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"8961c2ef138ffec779b0ea8fba27b43f","mimetype":"text/plain","path":"readme_habitat_suitability_maps.txt","platform_url":"https://ndownloader.figshare.com/files/17470232","sha1":"94a5c1a59b2d601e6cd831e0487406bf7b2342a9","sha256":"ecc1b13057fde871f7fce98ebdb3ed46c73bf5fa71f349085947191ba5b26acb","size":854,"status":"success"},{"md5":"6bee1894b6b53d73308d2b4a23ba2bd7","mimetype":"application/zip","path":"HabitatSuitabilityMaps.zip","platform_url":"https://ndownloader.figshare.com/files/17470235","sha1":"fab185662de454234ce205531b66d1f5b6294195","sha256":"3b5daffc265d55dd8c616c20c32b8fa42e44223e99527dc2d452b73d07a8825e","size":139735372,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"9751934","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.9751934.v1","ext_ids":{"doi":"10.6084/m9.figshare.9751934.v1"},"fatcat":{"release_ident":null,"work_ident":"dajvtbsy45a6hcjxyq4rncqfte"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.9751934.v1","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200229135146","terminal_sha1hex":"c89241bcc40ba1bf316dee75b0f4e8a6e098c445","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Current_and_projected_future_risks_of_freshwater_fish_invasions_in_China/9751934/1"},"total_size":139736226} +{"archiveorg_item_name":"dataverse.harvard.edu-7XCWTN-v5.0","cdx":{"datetime":"20200216081117","mimetype":"text/html","sha1b32":"YCHCKL3DSDKM5JBA7ZRC7UCQ6LVWCCUB","sha1hex":"c08e252f6390d4cea420fe622fd050f2eb610a81","status_code":200,"surt":"edu,harvard,dataverse)/dataset.xhtml?persistentid=doi:10.7910/dvn/7xcwtn","url":"https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/7XCWTN","warc_csize":24154,"warc_offset":134361015,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200216080230870-15086-15149-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200216080840222-15092-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":7,"file_meta":{"md5hex":"401cae5a3a5e27418f7b5947c8423d14","mimetype":"application/xhtml+xml","sha1hex":"c08e252f6390d4cea420fe622fd050f2eb610a81","sha256hex":"34dc22be03508b715a570a8503b3d69f74dc079a0752df8caa851cd45ede9842","size_bytes":189488},"hit":true,"hops":["https://doi.org/10.7910/dvn/7xcwtn","https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/7XCWTN"],"html_biblio":{"abstract":"Simulation data of an agent-based model of circular economy. See github repository : https://github.com/SFICSSS16-CircularEconomy/CircularEconomy","container_name":"Harvard Dataverse","contrib_names":["Broere, Joris","Moore, Christine","Raimbault, Juste","Serna, Jesus Mario","Somveille, Marius","Strombom, Evelyn","Sugar, Lorraine","Zhu, Ben"],"doi":"10.7910/DVN/7XCWTN","publisher":"Harvard Dataverse","release_date":"2020-02-13","title":"Simulation data for an Agent-based Model of Circular Economy"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"f4d197eb1c2073fec23ce60b5a555110","mimetype":"application/csv","path":"2018_06_16_21_01_13_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/CDCV3O&format=original","sha1":"56b0e0e4303d45a495ed234200f788249d399f05","sha256":"ab79ea86a9827083504b4eaf928c615eaeb5d21bae3e2849bc31e52887afa7d5","size":143635343,"status":"success"},{"extra":{"description":"Grid sampling"},"md5":"d487c9e903cec0f679e261a656789dfc","mimetype":"application/csv","path":"2018_06_18_12_12_05_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/FD9GXL&format=original","sha1":"a083f516dfb68980cc4ceb8b180944f5ef2f6593","sha256":"f41ce11ca6ea1ec29e57d6554fb24c36ad9170a72166abe4b67ec49f451feb7d","size":285512943,"status":"success"},{"extra":{"description":"","version":2},"md5":"1bb4e874728e07ca30ad0208cddfcc76","mimetype":"application/csv","path":"2018_06_19_15_02_56_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/JQ2OSU&format=original","sha1":"792b6542d248c75794745ebf42a6fdd142a37c36","sha256":"a04d4456429083feda2e5c58f0938aeaf747284504bdabcbc126b671ff45a900","size":83512721,"status":"success"},{"md5":"1c009129e505e7f017a55de3a2ce4b47","mimetype":"application/csv","path":"2018_06_19_18_50_44_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/VWMS6D&format=original","sha1":"fa5bb3bdc7c93ed8085b51e6f806d8ea89e8a8f2","sha256":"28e3b134c404ea335248dee9303f2359f30e73b95020727f1fb20deeafeef570","size":412020694,"status":"success"},{"md5":"ee1cccb6bbb42f559304cabd85976bd0","mimetype":"application/csv","path":"20180713_165922_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/FQDKLF&format=original","sha1":"f03af45c9a9fb9ae723fdff4894fd552ed3ef0db","sha256":"99a42b31f674511870e3fca5274aecbf6ee75a4183c01e5d275af9c6c7838251","size":17305211,"status":"success"},{"md5":"221a9740b173c1c806e59f254ecda188","mimetype":"application/csv","path":"20180713_210239_DIRECTSAMPLING_SYNTHETIC.csv","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/RL6EVR&format=original","sha1":"78dcc1f54c71ba4a3b4997417c4faf662a4ff3f8","sha256":"e58de20df2158999fa40aaa80ea38f1a67554e1d544489d38ba0457d23d6ac12","size":567459234,"status":"success"},{"extra":{"description":""},"md5":"cc194a3eae1ee6e226508ff3b7fc2d03","mimetype":"application/zip","path":"20180722_1631_NSGA2_SYNTHETIC_TRCOST3_DISTRIBSD0.01.zip","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/7XCWTN/NS3XTM","sha1":"a9e7e83be86a03e5de6d723b8578f9a9c8d34ec0","sha256":"f939fbf4c1aad1b36b05a3e7c41e2bea8512af2fd74832b5bf97b04baa82670d","size":361305971,"status":"success"}],"platform_base_url":null,"platform_domain":"dataverse.harvard.edu","platform_id":"doi:10.7910/DVN/7XCWTN","platform_name":"dataverse","request":{"base_url":"https://doi.org/10.7910/dvn/7xcwtn","ext_ids":{"doi":"10.7910/dvn/7xcwtn"},"fatcat":{"release_ident":null,"work_ident":"bmv3dcyso5dy5d2ghqar7tkci4"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.7910/dvn/7xcwtn","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200216081117","terminal_sha1hex":"c08e252f6390d4cea420fe622fd050f2eb610a81","terminal_status_code":200,"terminal_url":"https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/7XCWTN"},"total_size":1870752117} +{"archiveorg_item_name":"zenodo.org-1191135","cdx":{"datetime":"20200228180757","mimetype":"text/html","sha1b32":"SGXIZC56WIEK7BQ6Y6SRPCGSL3OE5DG7","sha1hex":"91ae8c8bbeb208af861ec7a51788d25edc4e8cdf","status_code":200,"surt":"org,zenodo)/record/1191135","url":"https://zenodo.org/record/1191135","warc_csize":9418,"warc_offset":11330533,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200228175520731-32472-32537-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200228180741865-32482-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"49df15db5b4bfec2003ca545e6654e3e","mimetype":"text/html","sha1hex":"91ae8c8bbeb208af861ec7a51788d25edc4e8cdf","sha256hex":"3443fa201ec6b217abf2e2b66894a504d1513101ed2991132394dcbcd4d9d107","size_bytes":38350},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1191135","https://zenodo.org/record/1191135"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Martineau, Celine N.","Nollen, Ellen A. A."],"doi":"10.5281/zenodo.1191135","release_date":"2018-03-06","title":"AQ2947 CGC N2 (Bristol, UK) | 2014-05-08T09:24:31+02:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"5414f8a7da96a06086f3c949b7a1da1f","mimetype":"application/x-hdf","path":"6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3_features.hdf5","platform_url":"https://zenodo.org/api/files/e0c4183e-2042-4b29-9f0b-cea1f29cc4a5/6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3_features.hdf5","sha1":"fd3faf5db88b9291de4cfb798c638f5dfc576166","sha256":"e4b021930653b031ce776f98a5b8ae5a325fec69d858c2042e6dbf9f9c7fe30e","size":13867721,"status":"success"},{"md5":"9845fa6c35f97808e97d0a5266f6049b","mimetype":"application/x-hdf","path":"6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3.hdf5","platform_url":"https://zenodo.org/api/files/e0c4183e-2042-4b29-9f0b-cea1f29cc4a5/6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3.hdf5","sha1":"4d13b7c563e78f8345f2f2da77084138258e6058","sha256":"25e8b5fffe4a6e117cfa30e878bdd6ce0c8c993f58159212732828de3c2f29b6","size":282718169,"status":"success"},{"md5":"f1de3f604e86ea09f0597bc5acd108ff","mimetype":"application/zip","path":"6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3.wcon.zip","platform_url":"https://zenodo.org/api/files/e0c4183e-2042-4b29-9f0b-cea1f29cc4a5/6bis_R_4_N2_7_Day16_CW_2014_05_08__09_24_31___1___3.wcon.zip","sha1":"591416c867dda3783ce27f84c27ce5fd07353001","sha256":"f302cb869722e7b88ea0ff345171dbc6f8389dc9ca975f9268d8d7bd7b42f43e","size":11930704,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1191135","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1191135","ext_ids":{"doi":"10.5281/zenodo.1191135"},"fatcat":{"release_ident":null,"work_ident":"ubwgzjsxgbhenc73efo5lnlnse"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1191135","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200228180757","terminal_sha1hex":"91ae8c8bbeb208af861ec7a51788d25edc4e8cdf","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1191135"},"total_size":308516594} +{"archiveorg_item_name":"figshare.com-7250108-v1","cdx":{"datetime":"20200214203509","mimetype":"text/html","sha1b32":"XWE4CXMWBYWNCMWTHUSHWE6NPBZ7RQZF","sha1hex":"bd89c15d960e2cd132d33d247b13cd7873f8c325","status_code":200,"surt":"com,figshare)/articles/zika_virus_genome_sequencing_directly_from_field_collected_mosquitoes/7250108/1","url":"https://figshare.com/articles/Zika_virus_genome_sequencing_directly_from_field_collected_mosquitoes/7250108/1","warc_csize":8001,"warc_offset":7292831,"warc_path":"OA-DOI-CRAWL-2020-02-20200214200644885-14616-14683-wbgrp-svc206/OA-DOI-CRAWL-2020-02-20200214203502551-14652-30393~wbgrp-svc206.us.archive.org~8443.warc.gz"},"file_count":13,"file_meta":{"md5hex":"1df69a9b75d7d3605d2cab6834f2c723","mimetype":"text/html","sha1hex":"bd89c15d960e2cd132d33d247b13cd7873f8c325","sha256hex":"9a934392db87d0b292a842a107d883f6d6a030e73a55e1531594ff142bccb9e8","size_bytes":46232},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.7250108.v1","https://figshare.com/articles/Zika_virus_genome_sequencing_directly_from_field_collected_mosquitoes/7250108/1"],"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"544f8c4577d6f32a5557828b0534ddbb","mimetype":"application/x-gzip","path":"962_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358639","sha1":"295d907283b11a8dd030580f77ffd42da9a205c1","sha256":"6c05587215f3c9581af0d234c452577f228480f6e24e35f3db93051fa1b9685e","size":33348539,"status":"success"},{"md5":"e978168811895c41cb66810c87c2d6eb","mimetype":"application/x-gzip","path":"961_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358642","sha1":"cf7fdb766dbac3a412117716fe3ba6089e933739","sha256":"a25e56e60aea04ccce2068dedf786cec95beb9945b67152bdc102fd0dfc643c5","size":35763356,"status":"success"},{"md5":"1c95335358c1a19ee2c7231952ccf88b","mimetype":"application/x-gzip","path":"959_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358645","sha1":"0d83e49b48d4f54afa92f98e451419be37f7da1b","sha256":"69e4288c3dd3af9a7edbdd57664e4b27ea929081c3dd231012065e6dc968ce82","size":20732751,"status":"success"},{"md5":"e31efa52e668745dfd63c89f935eabc7","mimetype":"application/x-gzip","path":"916_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358648","sha1":"f5930866cc65ffcf66815a2e7b83265e3f99aec6","sha256":"686933981e8541c99ecf0399558234e916387035004fb18bae45a0388289953a","size":73398433,"status":"success"},{"md5":"c5042647dea056f43bd592d5393173a2","mimetype":"application/x-gzip","path":"77_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358651","sha1":"7f75ac4158a4e31508cb9f985317e469c7004e53","sha256":"ced7a08fad2b1282e3e5aafcec60c21065979e0becae665751721d3734049653","size":34372469,"status":"success"},{"md5":"80a2c21c1be8069ff82205871cfa05e1","mimetype":"application/x-gzip","path":"39_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358654","sha1":"8ae3afe66f0278d9b2ef48a1f3f9cae856339cf1","sha256":"9074e73f3eede8b9b470c9a1e28aabc690f7c902b43469d88e0a264f05d2af27","size":21370025,"status":"success"},{"md5":"ffd1a05fcf679d59b5f47b2a9f20c28b","mimetype":"application/x-gzip","path":"28_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358660","sha1":"d83267c382639f38f2e34c126b5a6eb019a095a4","sha256":"063a71cb21b631db31172519762697accd0cf70bcd8df89a07e183c42f929ba3","size":118198979,"status":"success"},{"md5":"c86994562197e61ee22c795438879ae0","mimetype":"application/x-gzip","path":"1001_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358663","sha1":"daf4b3b910e27e9fd2d9867955a016ecbfc28936","sha256":"9dfe7eb3235c1d78e49776d92d08e7a63f894f10b17e27005aa73faf63b42588","size":25806420,"status":"success"},{"md5":"8ddfb853e4da3a70555ae7c81887c445","mimetype":"application/x-gzip","path":"01_A_probe_illumina_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358666","sha1":"37286ccc691ed0a9c62d1f543561a35b29fc6a61","sha256":"8dde3f1342406401edac2ee8425a1b6a7458d81142eb8b5491e4f6714068e67f","size":198985892,"status":"success"},{"md5":"b74c8d846eb5e472a0c7d3fe601c3ce4","mimetype":"application/x-gzip","path":"01_A_multiplex_PE243.sorted.bam","platform_url":"https://ndownloader.figshare.com/files/13358669","sha1":"520a6f6ab5ad7b101fd28d828b164fdad5d480b4","sha256":"24b5dbff7fe8dad8e9b1d2827779b38deb8a715aee39f9cc66441298aaaa0e1b","size":20688538,"status":"success"},{"md5":"2d83d490808b2dbc77ac8f39756e0f16","mimetype":"text/plain","path":"PE243.1.fasta","platform_url":"https://ndownloader.figshare.com/files/13358672","sha1":"5e6309d270af63f7b26fcbe513f37e2ca2db3f24","sha256":"0589add577629e28265c04b4b1aa3f3af81f260b0842d8619249b6d5e6dbf922","size":11042,"status":"success"},{"md5":"da97f0344da6dbfe556f8728d6bbb72a","mimetype":"application/x-gzip","path":"910_PE243.sorted_negative_pool_for_qPCR.bam","platform_url":"https://ndownloader.figshare.com/files/13358678","sha1":"d5ee498945bc63a5eef2d3945dd2dcb507a87abf","sha256":"c17a9ffebb17e5ea34bd22b2991728500496a825c89c60209b2f30c35bca1e57","size":17561088,"status":"success"},{"md5":"3d4a3977d997b2a6ed47b6293bb7a461","mimetype":"application/x-gzip","path":"1610_PE243.sorted_RNA_extraction_negative_control.bam","platform_url":"https://ndownloader.figshare.com/files/13358681","sha1":"7b2abaf6ccaa077d9d0f634e1800af04d52eafe9","sha256":"20ba55e6a253c294022ce42f3f77dac0cf7381fd4a26801e15629490562526e3","size":19591036,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"7250108","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.7250108.v1","ext_ids":{"doi":"10.6084/m9.figshare.7250108.v1"},"fatcat":{"release_ident":null,"work_ident":"ck7kunucardiddnuc5d54nik3e"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.7250108.v1","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200214203509","terminal_sha1hex":"bd89c15d960e2cd132d33d247b13cd7873f8c325","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/Zika_virus_genome_sequencing_directly_from_field_collected_mosquitoes/7250108/1"},"total_size":619828568} +{"archiveorg_item_name":"zenodo.org-1019361","cdx":{"datetime":"20200304095812","mimetype":"text/html","sha1b32":"6RNQEQNF7NQNXXGUB2BUNZ3UGZ2TDHBO","sha1hex":"f45b0241a5fb60dbdcd40e8346e7743675319c2e","status_code":200,"surt":"org,zenodo)/record/1019361","url":"https://zenodo.org/record/1019361","warc_csize":9740,"warc_offset":19216637,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200304084317906-36627-36693-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200304095720705-36680-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"e5ad3c5a9b67543dd374e723b4f89f42","mimetype":"text/html","sha1hex":"f45b0241a5fb60dbdcd40e8346e7743675319c2e","sha256hex":"4ab61f7285de4151caf7136a916e628755f37cb6b816556a977a127eb3d622f1","size_bytes":40544},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1019360","https://zenodo.org/record/1019361"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Javer, Avelino","Currie, Michael","Hokanson, Jim","Lee, Chee Wai","Li, Kezhi","Yemini, Eviatar","Grundy, Laura J","Li, Chris","Ch'ng, Quee-Lim","Schafer, William R","Kerr, Rex","Brown, André EX"],"doi":"10.5281/zenodo.1019361","release_date":"2017-10-20","title":"CB4852 C. elegans Wild Isolate, N3 (Rothamsted, UK) | 2011-03-28T12:12:19+01:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"983933f4d374f422fc3616eb192ff635","mimetype":"application/x-hdf","path":"135 CB4852 on food L_2011_03_28__12_12_19___7___1_features.hdf5","platform_url":"https://zenodo.org/api/files/7a190711-dfca-4cd2-827d-dc5dfc53d587/135%20CB4852%20on%20food%20L_2011_03_28__12_12_19___7___1_features.hdf5","sha1":"b7f3fbaee4c9574bf00e2325fe97b9a6fdc4f7a5","sha256":"317b97619c937eb93f728937bbc35f39777b377c9263452e7640837adfc23c10","size":19539202,"status":"success"},{"md5":"335ad5437c6b16b3ee186e9f93ae6605","mimetype":"application/x-hdf","path":"135 CB4852 on food L_2011_03_28__12_12_19___7___1.hdf5","platform_url":"https://zenodo.org/api/files/7a190711-dfca-4cd2-827d-dc5dfc53d587/135%20CB4852%20on%20food%20L_2011_03_28__12_12_19___7___1.hdf5","sha1":"b5eef215959837911dd3f20d8a2e4a43a90ce680","sha256":"68877164cf97ac659b7aad442f81a0d5800181354c2c689aa9541fbc42527b2e","size":289114919,"status":"success"},{"md5":"6222a8202972c32244c22155a27c4a8b","mimetype":"application/zip","path":"135 CB4852 on food L_2011_03_28__12_12_19___7___1.wcon.zip","platform_url":"https://zenodo.org/api/files/7a190711-dfca-4cd2-827d-dc5dfc53d587/135%20CB4852%20on%20food%20L_2011_03_28__12_12_19___7___1.wcon.zip","sha1":"7cdfa2980e1dd209a891aee4d7b9ad5bbf8de48d","sha256":"be69ef1ae5a404d913d99b68cad719c4ba210c0c0118425537250e5d33cd0124","size":18371729,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1019361","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1019360","ext_ids":{"core":"144855527","doi":"10.5281/zenodo.1019360"},"fatcat":{"release_ident":null,"work_ident":"rrcculd5bracnhk2iajnemeifa"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1019360","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200304095812","terminal_sha1hex":"f45b0241a5fb60dbdcd40e8346e7743675319c2e","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1019361"},"total_size":327025850} +{"archiveorg_item_name":"figshare.com-4531361-v1","cdx":{"datetime":"20200217084647","mimetype":"text/html","sha1b32":"UKVEGUVYL2YWL4FD66J57ZER74YZHZU3","sha1hex":"a2aa4352b85eb165f0a3f793dfe491ff3193e69b","status_code":200,"surt":"com,figshare)/articles/5_21_2016_il_50_/4531361/1","url":"https://figshare.com/articles/5_21_2016_IL_50_/4531361/1","warc_csize":6466,"warc_offset":54362629,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200217074559071-16586-16651-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200217084544583-16648-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":5,"file_meta":{"md5hex":"e0eae4f145d0e1167888ef11195b70c8","mimetype":"text/html","sha1hex":"a2aa4352b85eb165f0a3f793dfe491ff3193e69b","sha256hex":"9c51ef01262cc3b1271f97b05c30ca97419ea2d3d8c3bec5d45ca7dcb7782fd8","size_bytes":27767},"hit":true,"hops":["https://doi.org/10.6084/m9.figshare.4531361.v1","https://figshare.com/articles/5_21_2016_IL_50_/4531361/1"],"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"4cc81544024c79aa2df220e6adc7440a","mimetype":"audio/x-wav","path":"CR1_edit_af.wav","platform_url":"https://ndownloader.figshare.com/files/7337231","sha1":"8474f5079384aaad59684fe73f53d31ee1eda5e1","sha256":"e3e49c3efff4500e5a7bce89acd07e9c911ab949f936d92c7d47610ff5a99401","size":230400044,"status":"success"},{"md5":"f534513cbd371c6d4e95fdd114830196","mimetype":"audio/x-wav","path":"CR2_edit_af.wav","platform_url":"https://ndownloader.figshare.com/files/7337234","sha1":"7a646a29cf485511ac66aa7f4041db45519fdd66","sha256":"4ffba56506ebefff823650c8f1d5f080763f449dbf2d3ee1b155e2d23fff54a4","size":230400044,"status":"success"},{"md5":"f4db0d9181d2cc0510d97f1f6aecfa21","mimetype":"audio/x-wav","path":"TR1_edit_af.wav","platform_url":"https://ndownloader.figshare.com/files/7337237","sha1":"c4a55a68cf9d326bf44db5a018dff490b21147f7","sha256":"47d35e35b27cbd145380e84a3a6d9431bca5a5a290dbc3a532de7d24e698e747","size":230400044,"status":"success"},{"md5":"718a701c996ee41f48054627b14bc23c","mimetype":"audio/x-wav","path":"TR2_edit_af.wav","platform_url":"https://ndownloader.figshare.com/files/7337240","sha1":"5e144fa756ebf01de3dabf63327577ad9c6d7b60","sha256":"8d26397a36d447934a5539fd3304ceaa7b1bbaed65328f09a532dec1b8225b52","size":230400044,"status":"success"},{"md5":"a70a55f3212a866b14a23a1e379d89e5","mimetype":"text/plain","path":"Calibration.R","platform_url":"https://ndownloader.figshare.com/files/7950247","sha1":"f9bcd65e1e4b0892d6e4b8ac7855832495b30201","sha256":"d9c16aca86b5f97d3bf804fefe69fcc7cddf0c61751cac5d7fc41faa95ece83f","size":1371,"status":"success"}],"platform_base_url":null,"platform_domain":"figshare.com","platform_id":"4531361","platform_name":"figshare","request":{"base_url":"https://doi.org/10.6084/m9.figshare.4531361.v1","ext_ids":{"doi":"10.6084/m9.figshare.4531361.v1"},"fatcat":{"release_ident":null,"work_ident":"eak4mz7zgbbrbffsr5yezo62nm"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.6084/m9.figshare.4531361.v1","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200217084647","terminal_sha1hex":"a2aa4352b85eb165f0a3f793dfe491ff3193e69b","terminal_status_code":200,"terminal_url":"https://figshare.com/articles/5_21_2016_IL_50_/4531361/1"},"total_size":921601547} +{"archiveorg_item_name":"dataverse.harvard.edu-LULNUQ-v3.0","cdx":{"datetime":"20210318021952","mimetype":"text/html","sha1b32":"Q34MN2V2TXNNDWMWEA7YHLE4NDMJRFG5","sha1hex":"86f8c6eaba9ddad1d996203f83ac9c68d89894dd","status_code":200,"surt":"edu,harvard,dataverse)/dataset.xhtml?persistentid=doi:10.7910/dvn/lulnuq","url":"https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/LULNUQ","warc_csize":23496,"warc_offset":23414866,"warc_path":"EOT20-20210318020922-crawl865/EOT20-20210318021945-00093.warc.gz"},"file_count":3,"file_meta":{"md5hex":"7c859f9bfad8a918029b2af2e92d41b1","mimetype":"application/xhtml+xml","sha1hex":"86f8c6eaba9ddad1d996203f83ac9c68d89894dd","sha256hex":"c5dfe469e9fc729dda906fdb70f8df46a3157103b560c04779b2124eec62a480","size_bytes":164587},"hit":true,"hops":["https://doi.org/10.7910/dvn/lulnuq","https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/LULNUQ"],"html_biblio":{"abstract":"The data are packaged together into network common data form (netCDF) files. Three separate files are available, with data provided separately for each season: (1) best estimates of the generalized extreme value (GEV) distribution coefficients as well as bootstrap standard errors, (2) 10-, 20-, 50-, and 100-year return values (with bootstrap standard errors) for 1955, 1965, 1975, 1985, 1995, 2005, and 2015, and (3) smoothed bootstrap estimates (250 total) for each GEV coefficient (these are used to calculate the bootstrap standard errors in the first two files). We also provide code that can be used to calculate return values, return periods, or any function thereof for any year in 1950-2017, as well as the corresponding bootstrap sampling distribution (which can be used to quantify uncertainty via standard errors or confidence intervals).","container_name":"Harvard Dataverse","contrib_names":["Risser, Mark","Paciorek, Christopher","Wehner, Michael","O'Brien, Travis","Collins, William"],"doi":"10.7910/DVN/LULNUQ","publisher":"Harvard Dataverse","release_date":"2019-01-30","title":"A probabilistic gridded product for daily precipitation extremes over the United States"},"ingest_strategy":"archiveorg-fileset","manifest":[{"extra":{"description":"Smoothed bootstrap estimates (250 total) for each GEV coefficient (these are used to calculate the bootstrap standard errors in the first two files)"},"md5":"dd08020f907eff52b0bedf1f8c91b47f","mimetype":"application/octet-stream","path":"conus_gev_coefficients_boot.nc","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/LULNUQ/AMCAXG","sha1":"3ca7260ea88f32fac12cab01cacab8cc93ec157a","sha256":"b87a682a00bb6df746920d6290b3f9476aeb29f4730158166d9dfee0fabfaad8","size":358517156,"status":"success"},{"extra":{"description":"Best estimates of the GEV coefficients as well as bootstrap standard errors"},"md5":"908ccd87bc68f6c668c2a089c010a303","mimetype":"application/octet-stream","path":"conus_gev_coefficients.nc","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/LULNUQ/YEKIY9","sha1":"ad6ee5c20c742b15a0350a60d090c5fc1cfe83ab","sha256":"21d824ba8357ed1feedb4a366891ed854dbdad577e7699433f10c138ab3e8279","size":2872912,"status":"success"},{"extra":{"description":"10-, 20-, 50-, and 100-year return values (with bootstrap standard errors) for 1955, 1965, 1975, 1985, 1995, 2005, and 2015"},"md5":"00a5ea783f80703878576181b18b2ab3","mimetype":"application/octet-stream","path":"conus_return_values.nc","platform_url":"https://dataverse.harvard.edu/api/access/datafile/:persistentId/?persistentId=doi:10.7910/DVN/LULNUQ/6G69YR","sha1":"9627aed1c7a4bcd798f88f20bb51cc46127c597f","sha256":"40180fe9a9dfc02416334fea68523e76b1d4a4e5dd31acbf0d4ab783fdeb8f17","size":20081564,"status":"success"}],"platform_base_url":null,"platform_domain":"dataverse.harvard.edu","platform_id":"doi:10.7910/DVN/LULNUQ","platform_name":"dataverse","request":{"base_url":"https://doi.org/10.7910/dvn/lulnuq","ext_ids":{"doi":"10.7910/dvn/lulnuq"},"fatcat":{"release_ident":null,"work_ident":"cxysyusubbdynbrslg5p5rmaha"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.7910/dvn/lulnuq","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20210318021952","terminal_sha1hex":"86f8c6eaba9ddad1d996203f83ac9c68d89894dd","terminal_status_code":200,"terminal_url":"https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/LULNUQ"},"total_size":381471632} +{"archiveorg_item_name":"zenodo.org-1013264","cdx":{"datetime":"20200208232932","mimetype":"text/html","sha1b32":"LXYE7AVHJFGBGDMAADFSECD7I4VBKZSR","sha1hex":"5df04f82a7494c130d8000cb22087f472a156651","status_code":200,"surt":"org,zenodo)/record/1013264","url":"https://zenodo.org/record/1013264","warc_csize":9753,"warc_offset":6713945,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200208224749216-02851-02913-wbgrp-svc281/DATACITE-DOI-CRAWL-2020-01-20200208232930771-02907-3252~wbgrp-svc281.us.archive.org~8443.warc.gz"},"file_count":3,"file_meta":{"md5hex":"20cc1a277ff5dbc90f900af32a424d5f","mimetype":"text/html","sha1hex":"5df04f82a7494c130d8000cb22087f472a156651","sha256hex":"e2c73567328334088f4b1231af1f6b830e41cfdd1e6c06e57feeda8fd6a44223","size_bytes":40358},"hit":true,"hops":["https://doi.org/10.5281/zenodo.1013264","https://zenodo.org/record/1013264"],"html_biblio":{"container_name":"Zenodo","contrib_names":["Javer, Avelino","Currie, Michael","Hokanson, Jim","Lee, Chee Wai","Li, Kezhi","Yemini, Eviatar","Grundy, Laura J","Li, Chris","Ch'ng, Quee-Lim","Schafer, William R","Kerr, Rex","Brown, André EX"],"doi":"10.5281/zenodo.1013264","release_date":"2017-10-20","title":"QL53 ins-16(ok2919) | 2012-10-23T13:27:26+01:00"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"74251eb5b48a842ba10ba2a2456538e8","mimetype":"application/x-hdf","path":"ins-16 (ok2919) on food R_2012_10_23__13_27_26___7___13_features.hdf5","platform_url":"https://zenodo.org/api/files/2056b016-984d-415b-887c-3a65458eb75a/ins-16%20%28ok2919%29%20on%20food%20R_2012_10_23__13_27_26___7___13_features.hdf5","sha1":"29805568dde0aab56cfc5023c896135804aad1aa","sha256":"cac73972b3aa8d6315827a1421e999d48d2a0d0c784b202ff563f7604a0539b5","size":21517176,"status":"success"},{"md5":"61b1c96ce6129ecfd492683dc1f2576a","mimetype":"application/x-hdf","path":"ins-16 (ok2919) on food R_2012_10_23__13_27_26___7___13.hdf5","platform_url":"https://zenodo.org/api/files/2056b016-984d-415b-887c-3a65458eb75a/ins-16%20%28ok2919%29%20on%20food%20R_2012_10_23__13_27_26___7___13.hdf5","sha1":"25417ebe594128ce031ae948c0633fd545df91e9","sha256":"57c281a4a740b4d1ce8597c45b2f22cd94dfd05ae10c300b3ea16f6b7bdccf39","size":417904860,"status":"success"},{"md5":"f3778fcd2223c1814b72b5c7f12e27f5","mimetype":"application/zip","path":"ins-16 (ok2919) on food R_2012_10_23__13_27_26___7___13.wcon.zip","platform_url":"https://zenodo.org/api/files/2056b016-984d-415b-887c-3a65458eb75a/ins-16%20%28ok2919%29%20on%20food%20R_2012_10_23__13_27_26___7___13.wcon.zip","sha1":"722ac4939397a47a65b9463e99174287ffa07106","sha256":"f8beb3c103294b4e5d7de2cf43b5b416c43390b4175bcd3999426c9e950acb38","size":20771724,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"1013264","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.1013264","ext_ids":{"doi":"10.5281/zenodo.1013264"},"fatcat":{"release_ident":null,"work_ident":"i455la4stzdbflqghlspui5jpm"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.1013264","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200208232932","terminal_sha1hex":"5df04f82a7494c130d8000cb22087f472a156651","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/1013264"},"total_size":460193760} +{"archiveorg_item_name":"zenodo.org-2642028","cdx":{"datetime":"20200208182245","mimetype":"text/html","sha1b32":"U5ILWKL25FKYY3I7TC47LPRRLPMMZJPK","sha1hex":"a750bb297ae9558c6d1f98b9f5be315bd8cca5ea","status_code":200,"surt":"org,zenodo)/record/2642028","url":"https://zenodo.org/record/2642028","warc_csize":10101,"warc_offset":18042859,"warc_path":"DATACITE-DOI-CRAWL-2020-01-20200208175730712-02562-02621-wbgrp-svc280/DATACITE-DOI-CRAWL-2020-01-20200208182218221-02595-3211~wbgrp-svc280.us.archive.org~8443.warc.gz"},"file_count":30,"file_meta":{"md5hex":"3ca48231784254ce5c1585e549f9356d","mimetype":"text/html","sha1hex":"a750bb297ae9558c6d1f98b9f5be315bd8cca5ea","sha256hex":"a8e56e66344c6cbd1101315d5bcb396544e49b18eee43bd5f7fd67d3da482e8f","size_bytes":59352},"hit":true,"hops":["https://doi.org/10.5281/zenodo.2642028","https://zenodo.org/record/2642028"],"html_biblio":{"container_name":"Zenodo","contrib_names":["SXS Collaboration"],"doi":"10.5281/zenodo.2642028","release_date":"2018-04-28","title":"Binary black-hole simulation SXS:BBH:0359"},"ingest_strategy":"archiveorg-fileset","manifest":[{"md5":"feee69b232c5e560ae07c0056e51eeb3","mimetype":"text/plain","path":"common-metadata.txt","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/common-metadata.txt","sha1":"64a8e8f151c2c71f5dff4d7734a306beb10a5f39","sha256":"6aead76c4f13e4caf3b722d980dbb5cfae563ff673a4fc663bffd749b0e570e2","size":103,"status":"success"},{"md5":"a804365fb20b9699b0d1ddf58382d788","mimetype":"application/gzip","path":"EvID/ID_Files.tgz","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/EvID/ID_Files.tgz","sha1":"d4b281fc8c761672d3e34b26a37e9034eea33cf0","sha256":"e56e1ea137b0499c8444437ea91dbefdde8d2d1dfbe9310808d178a0b7b89330","size":14533887,"status":"success"},{"md5":"97df9e23d0203abe833d90192449357b","mimetype":"text/plain","path":"EvID/ID_Params.perl","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/EvID/ID_Params.perl","sha1":"b8f5a90c84f25cbd48490d1df3766b610824647d","sha256":"003914558b5d97d80e5a1264a9de2877bb958a44e011abeef7a52c4292733cb5","size":1576,"status":"success"},{"md5":"223dd02dfd7723d14a857bb4008e7a44","mimetype":"application/x-hdf","path":"Lev1/Horizons.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/Horizons.h5","sha1":"38a97225d196e445cc9aaebc27fc36563626d7cb","sha256":"43f310ec98a77eb6bfa4c0eac5be51f1e850e8098fe38cef7118e7765f2b328c","size":1776398,"status":"success"},{"md5":"bf6948dfd4ef6380ef2a15a7c4da8c0b","mimetype":"text/plain","path":"Lev1/metadata.json","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/metadata.json","sha1":"0013093eb4279904c2325b8cba6f2b6e75160f69","sha256":"a30e37708371e1f704d38043e2815a38fba4a8218d58e42f15fdb98a9290e53f","size":3672,"status":"success"},{"md5":"4c0bff862ba210f0ec246942178bf268","mimetype":"text/plain","path":"Lev1/metadata.txt","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/metadata.txt","sha1":"e48586dd22bfa32dfed37c13028c4d6ec97ca396","sha256":"2c3bbe611723ce7411f9113968b976c02d6539fedffec552d113eb44f1fb2fd6","size":4619,"status":"success"},{"md5":"1a7036551291add8413da607fdb4d6e8","mimetype":"application/x-hdf","path":"Lev1/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rh_FiniteRadii_CodeUnits.h5","sha1":"b7ed10c5c99ae0a415ac83a34dc35bee2a940515","sha256":"189847db6924b73914e963594e44226d348e69c5df3b0c26b20da66063fe4ac0","size":471173216,"status":"success"},{"md5":"e5944e73394cd0ea9e4f6580f73bb53a","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"f9d312ab06103c8ae50560c2ee2785f73d939566","sha256":"7d124d57501b3c8e06703d348b7283c5843ab918a53d3f3de8f6b91ec233bd2c","size":83015193,"status":"success"},{"md5":"29ba146f2d0fd36524aa7c0318aa3b4f","mimetype":"application/x-hdf","path":"Lev1/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"72c61c475decd48656736e9a8acfed6c5df8403c","sha256":"31f91464aa08c809bfd8562dae22bbd836a0ddee93bf33e2343fd4819c272bf4","size":85700127,"status":"success"},{"md5":"e99fd968479dd877faed7afec08de3d8","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"01414857f3fbb54296479f9c9edca87971fbd9e3","sha256":"658d114232d545abe12a88dd71c001e9ffccd7110c4f534fcb42334ff507f456","size":83599014,"status":"success"},{"md5":"7fae1b550b0c8a1a818f27c1a50f7ece","mimetype":"application/x-hdf","path":"Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"23d29763b04792a4694530d5e47b65c402fc3e69","sha256":"816fef488cf8f6139b7b5074fdf949b97b50224c6540b33a089ce61418c3f27d","size":85695581,"status":"success"},{"md5":"62ff0181c0fa6e17910dcdc2fdade15c","mimetype":"application/x-hdf","path":"Lev1/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev1/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"28192d2867f513446ffc07aee35fb1c230d8cedf","sha256":"d57e9efa779c8b0c69ff1a8be90247b8326f95a26606e28e9ef162d154ec772d","size":470051613,"status":"success"},{"md5":"c09d23393e4aba362f00d694ce1dd0d6","mimetype":"application/x-hdf","path":"Lev2/Horizons.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/Horizons.h5","sha1":"c27e027251ba32e9d3fba912d9f4a31455c9fce9","sha256":"8f877497f468b5c88f884a857c23f3e700c7caa3d7ec3054cf0f706f527961cb","size":1811820,"status":"success"},{"md5":"9a943d652e3ab2c8a270359f9c06619f","mimetype":"text/plain","path":"Lev2/metadata.json","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/metadata.json","sha1":"b298ab41fce625171e5422333f29f75e0171c7a9","sha256":"6db70dd90eba96c1c369d648442a979913a1abf0ed3c2b46f9d48ba73e2d9f0c","size":3728,"status":"success"},{"md5":"10c87ea02c76fc47d99de786dac4dc0f","mimetype":"text/plain","path":"Lev2/metadata.txt","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/metadata.txt","sha1":"e5ef356d4aa9ac7e0c4cafa34002659ff1cb973a","sha256":"52594c3f563a0f5025e17ae276341d5f489cf55eb7a95ec01e859b9e5015eff9","size":4651,"status":"success"},{"md5":"b75de95afcd1bb80db58b0e92e6f7ab8","mimetype":"application/x-hdf","path":"Lev2/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rh_FiniteRadii_CodeUnits.h5","sha1":"e48a136a5ba2aa9de5a2b9356d910c2cce7cceb1","sha256":"2aab888b96efe0737e3c61729a86925afa29c192edc3319c075883cdf4b02169","size":471976222,"status":"success"},{"md5":"6fc1c230660001f56047d7d46bc20b17","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"f968f6a50b18f88a4633b1f63c46a56601acbcb6","sha256":"c48ffa6e2ea0caea13c8d691238f9e0f6810d3a68ee2f75c6de88c8cd4d727b4","size":84084542,"status":"success"},{"md5":"dbef9cf1885ed1e784db214619658231","mimetype":"application/x-hdf","path":"Lev2/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"7f7200d411a39a96ec089573cee47d0d6f409e27","sha256":"d17d2f48f997a58718bd4ae67fab03e21e035ab5d7c973a786b58171f8e21f73","size":86637824,"status":"success"},{"md5":"124fd62e4abdf29ba663e796b19282d1","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"963d56cd37ca99fbb07f044b1972cde94693e10a","sha256":"a7f674bfd0e1637486f70f44f76fba4efc28e3ebacf718c15e0b3d7e09b024ae","size":84704300,"status":"success"},{"md5":"cd33f198476eb4e2c280c4699c3b3cfc","mimetype":"application/x-hdf","path":"Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"17c70fa9be5476b4d791b5a5d4299e01dc4df1ca","sha256":"f00c6816097ee0fd2d84113c948cd87ad8c1cb01f229d4ae7020b32e732f9f1d","size":86764582,"status":"success"},{"md5":"9d6c299de5f178bab12b47f3e5aa1bba","mimetype":"application/x-hdf","path":"Lev2/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev2/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"e9c2c0d598a2280ff0190d953790b09c35087e50","sha256":"3cead2865894547199a837cb3b1a6e98058523db764873af22592271d2679840","size":472946247,"status":"success"},{"md5":"0ea4fed8707f4c273377f00e0ae59dd0","mimetype":"application/x-hdf","path":"Lev3/Horizons.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/Horizons.h5","sha1":"53c49a7a6e81437af4a0dc4fa1d5525e1a4cc94e","sha256":"b786d09616ba37f09744770a450ca3925a5bb2a66acd1a5ce971fcddb691bf28","size":1778679,"status":"success"},{"md5":"21d9b2887a2fb8cbe91ad96dd4ddb3ed","mimetype":"text/plain","path":"Lev3/metadata.json","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/metadata.json","sha1":"30f2334f2e3f2206bba98feae121639ff89903a7","sha256":"b4fa72e1daa3ab13ca42f80cdeed6a39891f6d8553affa1fde46aee238426da9","size":3735,"status":"success"},{"md5":"3e594eb9d8e0be858d9c90f0e086be43","mimetype":"text/plain","path":"Lev3/metadata.txt","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/metadata.txt","sha1":"3ff49cad855bf3b08b636a497796938757d72e12","sha256":"d75daf0177705070f0ddab2574a3df0b4a15d845920162afce028792fc5671c7","size":4655,"status":"success"},{"md5":"b7e80cfcb1cad2774b4fa8deda88dec8","mimetype":"application/x-hdf","path":"Lev3/rh_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rh_FiniteRadii_CodeUnits.h5","sha1":"a36a9b495002b8d7a507acdb6172adb7e007bf0a","sha256":"c6195e83a0d0dc67400d804826b803bab02e7f111f2e8d27764390f909fa0df9","size":473899517,"status":"success"},{"md5":"a2aef8c5b324184873029ffb308ddfaa","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rhOverM_Asymptotic_GeometricUnits_CoM.h5","sha1":"8602a41b198c5c3b77682d1e24a31e59102f15aa","sha256":"46e957858bd28cb38fb7054b8700a0638c219632e58f957e07362e58a760697c","size":83801052,"status":"success"},{"md5":"3e4f34edc1720c81e9d1b03083f35182","mimetype":"application/x-hdf","path":"Lev3/rhOverM_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rhOverM_Asymptotic_GeometricUnits.h5","sha1":"24b4d5e81ae6d0e55d3e58b67a52d9847fd7abe4","sha256":"69ecd87d54c4c720b710402b9830f19d4e5567739fa8fecbde777757c00cc86f","size":86679940,"status":"success"},{"md5":"a3b62579228cd83670af46314ceb2269","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rMPsi4_Asymptotic_GeometricUnits_CoM.h5","sha1":"8765964464abab8cf39e89c7785cd3f33b204732","sha256":"23303dd4d1d28208ce45f2e06c211d09ad1e8e958d7f569badd2105bf67654e6","size":84711298,"status":"success"},{"md5":"e939b6a3eff045a83d6a592c8215ef45","mimetype":"application/x-hdf","path":"Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rMPsi4_Asymptotic_GeometricUnits.h5","sha1":"2626ab1261efe2cd40f2c6b5c80116b51bd995d1","sha256":"f590b180d23f3f7db93cca273318dc42b54f643d4ce0ff5f294f31865d6b38c9","size":86924324,"status":"success"},{"md5":"51ef298f5acc4ef12b32680d7e68c1be","mimetype":"application/x-hdf","path":"Lev3/rPsi4_FiniteRadii_CodeUnits.h5","platform_url":"https://zenodo.org/api/files/c2d76d00-fcad-423e-be5f-5b4bb324412e/Lev3/rPsi4_FiniteRadii_CodeUnits.h5","sha1":"90dfa3468f45625ab345a2d1661faa55bf709f87","sha256":"53d6bbce7aa3d77df32955b1483fc1086ec9e146a8e17af1e8f774301bf02ef5","size":473819309,"status":"success"}],"platform_base_url":null,"platform_domain":"zenodo.org","platform_id":"2642028","platform_name":"zenodo","request":{"base_url":"https://doi.org/10.5281/zenodo.2642028","ext_ids":{"doi":"10.5281/zenodo.2642028"},"fatcat":{"release_ident":null,"work_ident":"anhm43cbgfhbjp24yh4yu2m2oy"},"ingest_request_source":"fatcat-ingest","ingest_type":"dataset","link_source":"doi","link_source_id":"10.5281/zenodo.2642028","release_stage":"published"},"status":"success","terminal":{"terminal_dt":"20200208182245","terminal_sha1hex":"a750bb297ae9558c6d1f98b9f5be315bd8cca5ea","terminal_status_code":200,"terminal_url":"https://zenodo.org/record/2642028"},"total_size":3876111424} diff --git a/python/tests/import_doaj.py b/python/tests/import_doaj.py index 9c4ba552..81f17d7c 100644 --- a/python/tests/import_doaj.py +++ b/python/tests/import_doaj.py @@ -144,8 +144,7 @@ def test_doaj_dict_parse(doaj_importer): assert r.pages == "608-617" assert r.version is None assert r.language == "en" - # matched by ISSN, so wouldn't be defined normally - assert r.extra["container_name"] == "Materials & Design" + assert r.container_id assert len(r.abstracts) == 1 assert len(r.abstracts[0].content) == 1033 assert len(r.contribs) == 5 diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py index d9e7d294..65549d1d 100644 --- a/python/tests/import_ingest.py +++ b/python/tests/import_ingest.py @@ -5,6 +5,8 @@ from fixtures import * from fatcat_tools.importers import ( IngestFileResultImporter, + IngestFilesetFileResultImporter, + IngestFilesetResultImporter, IngestWebResultImporter, JsonLinePusher, ) @@ -20,6 +22,16 @@ def ingest_web_importer(api): yield IngestWebResultImporter(api) +@pytest.fixture(scope="function") +def ingest_fileset_importer(api): + yield IngestFilesetResultImporter(api) + + +@pytest.fixture(scope="function") +def ingest_fileset_file_importer(api): + yield IngestFilesetFileResultImporter(api) + + # TODO: use API to check that entities actually created... def test_ingest_importer_basic(ingest_importer): with open("tests/files/example_ingest.json", "r") as f: @@ -58,7 +70,6 @@ def test_ingest_importer_xml(ingest_importer): with open("tests/files/example_ingest_xml.json", "r") as f: ingest_importer.bezerk_mode = True counts = JsonLinePusher(ingest_importer, f).run() - print(counts) assert counts["insert"] == 1 assert counts["exists"] == 0 assert counts["skip"] == 0 @@ -86,7 +97,6 @@ def test_ingest_importer_web(ingest_web_importer): with open("tests/files/example_ingest_html.json", "r") as f: ingest_web_importer.bezerk_mode = True counts = JsonLinePusher(ingest_web_importer, f).run() - print(counts) assert counts["insert"] == 1 assert counts["exists"] == 0 assert counts["skip"] == 0 @@ -139,7 +149,6 @@ def test_ingest_importer_stage(ingest_importer, api): ingest_importer.reset() ingest_importer.push_record(raw) counts = ingest_importer.finish() - print(counts) assert counts["total"] == 1 assert counts[row["status"]] == 1 @@ -182,3 +191,118 @@ def test_ingest_dict_parse_old(ingest_importer): if u.rel == "webarchive": assert u.url.startswith("https://web.archive.org/") assert len(f.release_ids) == 1 + + +def test_ingest_fileset_dict_parse(ingest_fileset_importer): + with open("tests/files/example_fileset_ingest_result.json", "r") as f: + raw = json.loads(f.readline()) + fs = ingest_fileset_importer.parse_record(raw) + assert len(fs.manifest) == 3 + assert fs.manifest[0].sha1 == "c0669e84e7b9052cc0f342e8ce7d31d59956326a" + assert fs.manifest[0].md5 == "caf4d9fc2c6ebd0d9251ac84e0b6b006" + assert fs.manifest[0].mimetype == "application/x-hdf" + assert fs.manifest[0].size == 16799750 + assert fs.manifest[0].path == "N2 on food R_2010_03_25__10_53_27___4___1_features.hdf5" + assert ( + fs.manifest[0].extra["original_url"] + == "https://zenodo.org/api/files/563203f6-6de5-46d9-b305-ba42604f2508/N2%20on%20food%20R_2010_03_25__10_53_27___4___1_features.hdf5" + ) + assert len(fs.urls) == 2 + matched = 0 + for u in fs.urls: + if u.rel == "web": + assert u.url == "https://zenodo.org/record/1028059" + matched += 1 + if u.rel == "archive-base": + assert u.url == "https://archive.org/download/zenodo.org-1028059/" + matched += 1 + assert matched == 2 + assert len(fs.release_ids) == 1 + + +def test_ingest_fileset_importer(ingest_fileset_importer): + last_index = ingest_fileset_importer.api.get_changelog(limit=1)[0].index + with open("tests/files/example_fileset_ingest_result.json", "r") as f: + ingest_fileset_importer.bezerk_mode = True + counts = JsonLinePusher(ingest_fileset_importer, f).run() + assert counts["insert"] == 7 + assert counts["exists"] == 0 + assert counts["skip"] == 13 + assert counts["skip-release-not-found"] == 13 + + # fetch most recent editgroup + change = ingest_fileset_importer.api.get_changelog_entry(index=last_index + 1) + eg = change.editgroup + assert eg.description + assert "filesets crawled from web" in eg.description.lower() + assert eg.extra["git_rev"] + assert "fatcat_tools.IngestFilesetResultImporter" in eg.extra["agent"] + + # re-insert; should skip + with open("tests/files/example_fileset_ingest_result.json", "r") as f: + ingest_fileset_importer.reset() + ingest_fileset_importer.bezerk_mode = False + counts = JsonLinePusher(ingest_fileset_importer, f).run() + + assert counts["insert"] == 0 + assert counts["exists"] == 7 + assert counts["skip"] == 13 + assert counts["skip-release-not-found"] == 13 + + +def test_ingest_fileset_file_dict_parse(ingest_fileset_file_importer): + with open("tests/files/example_fileset_file_ingest_result.json", "r") as f: + raw = json.loads(f.readline()) + fe = ingest_fileset_file_importer.parse_record(raw) + assert fe.sha1 == "6fb020064da66bb7a666c17555611cf6820fc9ae" + assert fe.md5 == "dfc41b617564f99a12e6077a6208876f" + assert fe.sha256 == "2febad53ff0f163a18d7cbb913275bf99ed2544730cda191458837e2b0da9d18" + assert fe.mimetype == "image/tiff" + assert fe.size == 410631015 + assert fe.extra["path"] == "NDVI_Diff_1990_2018_T06.tif" + assert len(fe.urls) == 2 + matched = 0 + for u in fe.urls: + if u.rel == "web": + assert u.url == "https://ndownloader.figshare.com/files/14460875" + matched += 1 + if u.rel == "archive": + assert ( + u.url + == "https://archive.org/download/springernature.figshare.com-7767695-v1/NDVI_Diff_1990_2018_T06.tif" + ) + matched += 1 + assert matched == 2 + assert len(fe.release_ids) == 1 + + +def test_ingest_fileset_file_importer(ingest_fileset_file_importer): + """ + Similar to the above, but specifically tests 'file'/'success-file' import pathway + """ + last_index = ingest_fileset_file_importer.api.get_changelog(limit=1)[0].index + with open("tests/files/example_fileset_file_ingest_result.json", "r") as f: + ingest_fileset_file_importer.bezerk_mode = True + counts = JsonLinePusher(ingest_fileset_file_importer, f).run() + assert counts["insert"] == 16 + assert counts["exists"] == 0 + assert counts["skip"] == 4 + assert counts["skip-bad-hashes"] == 4 + + # fetch most recent editgroup + change = ingest_fileset_file_importer.api.get_changelog_entry(index=last_index + 1) + eg = change.editgroup + assert eg.description + assert "crawled from web" in eg.description.lower() + assert eg.extra["git_rev"] + assert "fatcat_tools.IngestFilesetFileResultImporter" in eg.extra["agent"] + + # re-insert; should skip + with open("tests/files/example_fileset_file_ingest_result.json", "r") as f: + ingest_fileset_file_importer.reset() + ingest_fileset_file_importer.bezerk_mode = False + counts = JsonLinePusher(ingest_fileset_file_importer, f).run() + assert counts["insert"] == 0 + assert counts["exists"] == 16 + assert counts["skip"] == 4 + assert counts["skip-bad-hashes"] == 4 diff --git a/python/tests/web_citation_csl.py b/python/tests/web_citation_csl.py index e3dbec2b..9dfe94f6 100644 --- a/python/tests/web_citation_csl.py +++ b/python/tests/web_citation_csl.py @@ -54,8 +54,8 @@ def test_release_bibtex(app, api): rv = app.get("/release/{}".format(r1edit.ident)) assert rv.status_code == 200 assert b"BibTeX" not in rv.data - with pytest.raises(ValueError): - rv = app.get("/release/{}.bib".format(r1edit.ident)) + rv = app.get("/release/{}.bib".format(r1edit.ident)) + assert rv.status_code == 400 # create release can have citeproc run on it (no authors) eg = quick_eg(api) diff --git a/rust/.gitignore b/rust/.gitignore index 03e50598..f237ae31 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,6 +1,5 @@ .env target/ -bin/ fatcat-*.tar.gz !.cargo diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6fa62c5f..56606fc6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,57 +4,66 @@ version = 3 [[package]] name = "addr2line" -version = "0.14.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9a933f4e58658d7b12defcf96dc5c720f20832deebe3e0a19efd3b6aaeeb9e" +checksum = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" dependencies = [ - "memchr 2.3.4", + "memchr 2.5.0", ] [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +dependencies = [ + "memchr 2.5.0", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "memchr 2.3.4", + "libc", ] [[package]] name = "ansi_term" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] -name = "antidote" -version = "1.0.0" +name = "arc-swap" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" +checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "arrayvec" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" dependencies = [ "nodrop", ] @@ -67,34 +76,38 @@ checksum = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50" [[package]] name = "atty" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ + "hermit-abi", "libc", - "termion", - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] name = "autocfg" -version = "0.1.2" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" dependencies = [ "addr2line", + "cc", "cfg-if 1.0.0", "libc", "miniz_oxide", @@ -118,7 +131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" dependencies = [ "byteorder", - "safemem 0.3.0", + "safemem 0.3.3", ] [[package]] @@ -131,10 +144,16 @@ dependencies = [ ] [[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] name = "bitflags" -version = "1.0.4" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bodyparser" @@ -142,11 +161,11 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f023abfa58aad6f6bc4ae0630799e24d5ee0ab8bb2e49f651d9b1f9aa4f52f30" dependencies = [ - "iron 0.6.0", + "iron 0.6.1", "persistent", "plugin", - "serde 1.0.117", - "serde_json 1.0.38", + "serde", + "serde_json", ] [[package]] @@ -160,18 +179,25 @@ dependencies = [ ] [[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + +[[package]] name = "byteorder" -version = "1.3.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ade3d27603c2cb345eb0912aec461a6dec7e06a4ae48589904e808335c7afa" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" dependencies = [ "byteorder", + "either", "iovec", ] @@ -181,14 +207,20 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3970b43fb5b4049ac086516d1ca06f929924aa0ed38566ba52d6cbb291892a40" dependencies = [ - "crossbeam 0.3.2", + "crossbeam", ] [[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] name = "cfg-if" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "cfg-if" @@ -203,19 +235,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" dependencies = [ "num", - "time", + "time 0.1.44", ] [[package]] name = "chrono" -version = "0.4.6" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ + "iana-time-zone", + "js-sys", "num-integer", - "num-traits 0.2.6", - "serde 1.0.117", - "time", + "num-traits", + "serde", + "time 0.1.44", + "wasm-bindgen", + "winapi 0.3.9", ] [[package]] @@ -226,9 +262,9 @@ checksum = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" [[package]] name = "clap" -version = "2.32.0" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", @@ -263,7 +299,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" dependencies = [ - "time", + "time 0.1.44", "url 1.7.2", ] @@ -276,53 +312,37 @@ dependencies = [ "cookie", "failure", "idna 0.1.5", - "log 0.4.11", + "log 0.4.17", "publicsuffix", - "serde 1.0.117", - "serde_json 1.0.38", - "time", + "serde", + "serde_json", + "time 0.1.44", "try_from", "url 1.7.2", ] [[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "crossbeam" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd66663db5a988098a89599d4857919b3acf7f61402e61365acfd3919857b9be" - -[[package]] -name = "crossbeam" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" [[package]] -name = "crossbeam" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad4c7ea749d9fb09e23c5cb17e3b70650860553a0e2744e38446b1803bf7db94" -dependencies = [ - "cfg-if 0.1.6", - "crossbeam-channel 0.3.8", - "crossbeam-deque", - "crossbeam-epoch 0.7.1", - "crossbeam-utils 0.6.5", - "lazy_static 1.4.0", - "num_cpus", - "parking_lot 0.7.1", -] - -[[package]] name = "crossbeam-channel" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -332,27 +352,28 @@ dependencies = [ "crossbeam-utils 0.5.0", "parking_lot 0.6.4", "rand 0.5.6", - "smallvec", + "smallvec 0.6.14", ] [[package]] name = "crossbeam-channel" -version = "0.3.8" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0ed1a4de2235cabda8558ff5840bffb97fcb64c97827f354a451307df5f72b" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ - "crossbeam-utils 0.6.5", - "smallvec", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", ] [[package]] name = "crossbeam-deque" -version = "0.6.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" dependencies = [ - "crossbeam-epoch 0.7.1", - "crossbeam-utils 0.6.5", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -362,25 +383,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2449aaa4ec7ef96e5fb24db16024b935df718e9ae1cec0a1e68feeca2efca7b8" dependencies = [ "arrayvec", - "cfg-if 0.1.6", - "crossbeam-utils 0.6.5", + "cfg-if 0.1.10", + "crossbeam-utils 0.6.6", "lazy_static 1.4.0", - "memoffset", - "scopeguard", + "memoffset 0.2.1", + "scopeguard 0.3.3", ] [[package]] name = "crossbeam-epoch" -version = "0.7.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c9e3102cc2d69cd681412141b390abd55a362afc1540965dad0ad4d34280b4" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ - "arrayvec", - "cfg-if 0.1.6", - "crossbeam-utils 0.6.5", + "autocfg 1.1.0", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", "lazy_static 1.4.0", - "memoffset", - "scopeguard", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard 1.1.0", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -391,19 +424,40 @@ checksum = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" [[package]] name = "crossbeam-utils" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 0.1.10", "lazy_static 1.4.0", ] [[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.1.0", + "cfg-if 0.1.10", + "lazy_static 1.4.0", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + +[[package]] name = "data-encoding" -version = "2.1.2" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f47ca1860a761136924ddd2422ba77b2ea54fe8cc75b9040804a0d9d32ad97" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" [[package]] name = "debugid" @@ -412,36 +466,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "088c9627adec1e494ff9dea77377f1e69893023d631254a0ec68b16ee20be3e9" dependencies = [ "lazy_static 1.4.0", - "regex 1.4.2", - "serde 1.0.117", - "uuid 0.7.2", + "regex 1.6.0", + "serde", + "uuid 0.7.4", ] [[package]] name = "diesel" -version = "1.4.1" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2469cbcf1dfb9446e491cac4c493c2554133f87f7d041e892ac82e5cd36e863" +checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" dependencies = [ "bitflags", "byteorder", - "chrono 0.4.6", + "chrono 0.4.22", "diesel_derives", "pq-sys", "r2d2", - "serde_json 1.0.38", + "serde_json", "uuid 0.6.5", ] [[package]] name = "diesel_derives" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62a27666098617d52c487a41f70de23d44a1dc1f3aa5877ceba2790fb1f1cab4" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ - "proc-macro2 0.4.27", - "quote 0.6.11", - "syn 0.15.26", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -455,6 +509,27 @@ dependencies = [ ] [[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] name = "dotenv" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -465,9 +540,24 @@ dependencies = [ [[package]] name = "dtoa" -version = "0.4.3" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "ed25519" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369" +dependencies = [ + "signature", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "encoding" @@ -535,23 +625,23 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "encoding_rs" -version = "0.8.15" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd251508d65030820f3a4317af2248180db337fdb25d89967956242580277813" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 1.0.0", ] [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ "atty", "humantime", - "log 0.4.11", - "regex 1.4.2", + "log 0.4.17", + "regex 1.6.0", "termcolor", ] @@ -575,19 +665,10 @@ dependencies = [ ] [[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check 0.9.2", -] - -[[package]] name = "failure" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" dependencies = [ "backtrace", "failure_derive", @@ -595,13 +676,13 @@ dependencies = [ [[package]] name = "failure_derive" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 0.4.27", - "quote 0.6.11", - "syn 0.15.26", + "proc-macro2", + "quote", + "syn", "synstructure", ] @@ -610,7 +691,7 @@ name = "fatcat" version = "0.5.0" dependencies = [ "cadence", - "chrono 0.4.6", + "chrono 0.4.22", "clap", "crossbeam-channel 0.2.6", "data-encoding", @@ -621,21 +702,21 @@ dependencies = [ "failure", "fatcat-openapi", "futures", - "hyper 0.10.15", - "iron 0.6.0", + "hyper 0.10.16", + "iron 0.6.1", "iron-slog", "iron-test", "lazy_static 1.4.0", - "log 0.3.9", + "log 0.4.17", "macaroon", "num_cpus", - "rand 0.3.23", - "regex 1.4.2", + "rand 0.8.5", + "regex 1.6.0", "sentry", - "serde 1.0.117", - "serde_derive 1.0.117", + "serde", + "serde_derive", "serde_ignored", - "serde_json 1.0.38", + "serde_json", "sha1", "slog", "slog-async", @@ -651,20 +732,20 @@ name = "fatcat-openapi" version = "0.5.0" dependencies = [ "bodyparser", - "chrono 0.4.6", + "chrono 0.4.22", "clap", - "error-chain 0.11.0", + "error-chain", "futures", - "hyper 0.10.15", - "iron 0.6.0", + "hyper 0.10.16", + "iron 0.6.1", "lazy_static 1.4.0", - "log 0.4.11", + "log 0.4.17", "multipart", "router", - "serde 1.0.117", - "serde_derive 1.0.117", + "serde", + "serde_derive", "serde_ignored", - "serde_json 1.0.38", + "serde_json", "swagger", "url 1.7.2", "urlencoded", @@ -673,37 +754,34 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.19" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if 1.0.0", "crc32fast", - "libc", "miniz_oxide", ] [[package]] name = "fnv" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", - "percent-encoding 2.1.0", + "percent-encoding 2.2.0", ] [[package]] name = "fuchsia-cprng" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f7f8eb465745ea9b02e2704612a9946a59fa40572086c6fd49d6ddcf30bf31" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "fuchsia-zircon" @@ -723,9 +801,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.1.25" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures-cpupool" @@ -738,16 +816,27 @@ dependencies = [ ] [[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] name = "gimli" -version = "0.23.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" [[package]] name = "h2" -version = "0.1.16" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb2b25a33e231484694267af28fec74ac63b5ccf51ee2065a5e313b834d836e" +checksum = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" dependencies = [ "byteorder", "bytes", @@ -755,28 +844,55 @@ dependencies = [ "futures", "http", "indexmap", - "log 0.4.11", + "log 0.4.17", "slab", "string", "tokio-io", ] [[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] name = "http" -version = "0.1.15" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a10e5b573b9a0146545010f50772b9e8b1dd0a256564cc4307694c68832a2f5" +checksum = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0" dependencies = [ "bytes", "fnv", - "itoa 0.4.3", + "itoa 0.4.8", +] + +[[package]] +name = "http-body" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" +dependencies = [ + "bytes", + "futures", + "http", + "tokio-buf", ] [[package]] name = "httparse" -version = "1.3.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -786,15 +902,15 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.10.15" +version = "0.10.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0caae6b71d266b91b4a83111a61d2b94ed2e2bea024c532b933dcff867e58c" +checksum = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" dependencies = [ "base64 0.9.3", "httparse", @@ -802,7 +918,7 @@ dependencies = [ "log 0.3.9", "mime 0.2.6", "num_cpus", - "time", + "time 0.1.44", "traitobject", "typeable", "unicase 1.4.2", @@ -811,22 +927,25 @@ dependencies = [ [[package]] name = "hyper" -version = "0.12.23" +version = "0.12.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "860faf61a9957c9cb0e23e69f1c8290e92f6eb660fcdd1f2d6777043a2ae1a46" +checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52" dependencies = [ "bytes", "futures", "futures-cpupool", "h2", "http", + "http-body", "httparse", "iovec", - "itoa 0.4.3", - "log 0.4.11", + "itoa 0.4.8", + "log 0.4.17", "net2", - "time", + "rustc_version", + "time 0.1.44", "tokio", + "tokio-buf", "tokio-executor", "tokio-io", "tokio-reactor", @@ -837,6 +956,20 @@ dependencies = [ ] [[package]] +name = "iana-time-zone" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "once_cell", + "wasm-bindgen", + "winapi 0.3.9", +] + +[[package]] name = "idna" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -849,9 +982,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -859,6 +992,16 @@ dependencies = [ ] [[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] name = "im" version = "12.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -871,18 +1014,21 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.0.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg 1.1.0", + "hashbrown", +] [[package]] name = "iovec" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" dependencies = [ "libc", - "winapi 0.2.8", ] [[package]] @@ -893,7 +1039,7 @@ checksum = "2440ae846e7a8c7f9b401db8f6e31b4ea5e7d3688b91761337da7e054520c75b" dependencies = [ "conduit-mime-types", "error", - "hyper 0.10.15", + "hyper 0.10.16", "lazy_static 0.2.11", "log 0.3.9", "modifier", @@ -905,13 +1051,13 @@ dependencies = [ [[package]] name = "iron" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8e17268922834707e1c29e8badbf9c712c9c43378e1b6a3388946baff10be2" +checksum = "c6d308ca2d884650a8bf9ed2ff4cb13fbb2207b71f64cda11dc9b892067295e8" dependencies = [ - "hyper 0.10.15", + "hyper 0.10.16", "log 0.3.9", - "mime_guess 1.8.6", + "mime_guess 1.8.8", "modifier", "num_cpus", "plugin", @@ -925,8 +1071,8 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41941d50d2c936a4c609f457ae8821f9888aa6ed8641f5d6e5d9b22e15041255" dependencies = [ - "chrono 0.4.6", - "iron 0.6.0", + "chrono 0.4.22", + "iron 0.6.1", "slog", ] @@ -936,36 +1082,33 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1944bcf30f8b3f51ebf01e715517dd9755e9480934778d6de70179a41d283c1" dependencies = [ - "hyper 0.10.15", - "iron 0.6.0", + "hyper 0.10.16", + "iron 0.6.1", "log 0.3.9", "url 1.7.2", "uuid 0.5.1", ] [[package]] -name = "isatty" -version = "0.1.9" +name = "itoa" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31a8281fc93ec9693494da65fbf28c0c2aa60a2eaec25dc58e2f31952e95edc" -dependencies = [ - "cfg-if 0.1.6", - "libc", - "redox_syscall", - "winapi 0.3.6", -] +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "0.3.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] -name = "itoa" -version = "0.4.3" +name = "js-sys" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] [[package]] name = "kernel32-sys" @@ -996,25 +1139,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "lazycell" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" - -[[package]] name = "libc" -version = "0.2.80" +version = "0.2.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" [[package]] name = "libsodium-sys" -version = "0.0.13" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e92532ef19ec2da77f5a89ae63a5c3dbb5136e8dada4e2c278107c1e1c773d8" +checksum = "6b779387cd56adfbc02ea4a668e704f729be8d6a6abd2c27ca5ee537849a92fd" dependencies = [ + "cc", "libc", "pkg-config", + "walkdir", ] [[package]] @@ -1024,7 +1163,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" dependencies = [ "owning_ref", - "scopeguard", + "scopeguard 0.3.3", +] + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard 1.1.0", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg 1.1.0", + "scopeguard 1.1.0", ] [[package]] @@ -1033,36 +1191,41 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.11", + "log 0.4.17", ] [[package]] name = "log" -version = "0.4.11" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 1.0.0", ] [[package]] name = "macaroon" -version = "0.1.1" -source = "git+https://github.com/bnewbold/libmacaroon-rs?branch=bnewbold-broken#346b4bb21c79958dde301501083bfdaa7aa83f73" +version = "0.2.0" +source = "git+https://github.com/bnewbold/macaroon?branch=bnewbold-legacy-hmac-fixed-branch#d0edb6c7e5f5893660d2db3a5cfce26c6e57f6a8" dependencies = [ - "log 0.3.9", - "rustc-serialize", - "serde 0.9.15", - "serde_derive 0.9.15", - "serde_json 0.9.10", + "base64 0.13.0", + "log 0.4.17", + "serde", + "serde_json", "sodiumoxide", ] [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" @@ -1075,9 +1238,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -1086,23 +1249,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" [[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] name = "migrations_internals" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8089920229070f914b9ce9b07ef60e175b2b9bc2d35c3edd8bf4433604e863b9" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" dependencies = [ "diesel", ] [[package]] name = "migrations_macros" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1664412abf7db2b8a6d58be42a38b099780cc542b5b350383b805d88932833fe" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" dependencies = [ "migrations_internals", - "quote 0.3.15", - "syn 0.11.11", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1116,18 +1289,15 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.13" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e27ca21f40a310bd06d9031785f4801710d566c184a6e15bad4f1d9b65f9425" -dependencies = [ - "unicase 2.6.0", -] +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "1.8.6" +version = "1.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4c0961143b8efdcfa29c3ae63281601b446a4a668165454b6c90f8024954c5" +checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" dependencies = [ "mime 0.2.6", "phf", @@ -1137,37 +1307,36 @@ dependencies = [ [[package]] name = "mime_guess" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ - "mime 0.3.13", + "mime 0.3.16", "unicase 2.6.0", ] [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" dependencies = [ "adler", - "autocfg 1.0.1", ] [[package]] name = "mio" -version = "0.6.16" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" dependencies = [ + "cfg-if 0.1.10", "fuchsia-zircon", "fuchsia-zircon-sys", "iovec", "kernel32-sys", - "lazycell", "libc", - "log 0.4.11", + "log 0.4.17", "miow", "net2", "slab", @@ -1176,9 +1345,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" dependencies = [ "kernel32-sys", "net2", @@ -1200,11 +1369,11 @@ checksum = "92f54eb45230c3aa20864ccf0c277eeaeadcf5e437e91731db498dbf7fbe0ec6" dependencies = [ "buf_redux", "httparse", - "hyper 0.10.15", - "iron 0.6.0", + "hyper 0.10.16", + "iron 0.6.1", "log 0.3.9", "mime 0.2.6", - "mime_guess 1.8.6", + "mime_guess 1.8.8", "rand 0.3.23", "safemem 0.2.0", "tempdir", @@ -1214,20 +1383,20 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.33" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 0.1.10", "libc", - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] name = "nodrop" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] name = "num" @@ -1237,63 +1406,78 @@ checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" dependencies = [ "num-integer", "num-iter", - "num-traits 0.2.6", + "num-traits", ] [[package]] name = "num-integer" -version = "0.1.39" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "num-traits 0.2.6", + "autocfg 1.1.0", + "num-traits", ] [[package]] name = "num-iter" -version = "0.1.37" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3fdbbc3291a5464dc57b03860ec37ca6bf915ed6ee385e7c6c052c422b2124" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ + "autocfg 1.1.0", "num-integer", - "num-traits 0.2.6", + "num-traits", ] [[package]] name = "num-traits" -version = "0.1.43" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "num-traits 0.2.6", + "autocfg 1.1.0", ] [[package]] -name = "num-traits" -version = "0.2.6" +name = "num_cpus" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] [[package]] -name = "num_cpus" -version = "1.9.0" +name = "num_threads" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a69d464bdc213aaaff628444e99578ede64e9c854025aa43b9796530afa9238" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.22.0" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr 2.5.0", +] + +[[package]] +name = "once_cell" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "owning_ref" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" +checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" dependencies = [ "stable_deref_trait", ] @@ -1304,18 +1488,29 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5" dependencies = [ - "lock_api", + "lock_api 0.1.5", "parking_lot_core 0.3.1", ] [[package]] name = "parking_lot" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" dependencies = [ - "lock_api", - "parking_lot_core 0.4.0", + "lock_api 0.3.4", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api 0.4.9", + "parking_lot_core 0.9.3", ] [[package]] @@ -1327,21 +1522,36 @@ dependencies = [ "libc", "rand 0.5.6", "rustc_version", - "smallvec", - "winapi 0.3.6", + "smallvec 0.6.14", + "winapi 0.3.9", ] [[package]] name = "parking_lot_core" -version = "0.4.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" dependencies = [ + "cfg-if 0.1.10", + "cloudabi", "libc", - "rand 0.6.5", + "redox_syscall 0.1.57", "rustc_version", - "smallvec", - "winapi 0.3.6", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.2.16", + "smallvec 1.9.0", + "windows-sys", ] [[package]] @@ -1352,9 +1562,9 @@ checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "persistent" @@ -1362,7 +1572,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e8fa0009c4f3d350281309909c618abddf10bb7e3145f28410782f6a5ec74c5" dependencies = [ - "iron 0.6.0", + "iron 0.6.1", "plugin", ] @@ -1407,9 +1617,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.14" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plugin" @@ -1421,77 +1631,56 @@ dependencies = [ ] [[package]] -name = "pq-sys" -version = "0.4.6" +name = "ppv-lite86" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" -dependencies = [ - "vcpkg", -] +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] -name = "proc-macro2" -version = "0.4.27" +name = "pq-sys" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" +checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" dependencies = [ - "unicode-xid 0.1.0", + "vcpkg", ] [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" dependencies = [ - "unicode-xid 0.2.1", + "unicode-ident", ] [[package]] name = "publicsuffix" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" -dependencies = [ - "error-chain 0.12.4", - "idna 0.2.0", - "lazy_static 1.4.0", - "regex 1.4.2", - "url 2.2.0", -] - -[[package]] -name = "quote" -version = "0.3.15" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" - -[[package]] -name = "quote" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" +checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ - "proc-macro2 0.4.27", + "idna 0.2.3", + "url 2.3.1", ] [[package]] name = "quote" -version = "1.0.7" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2", ] [[package]] name = "r2d2" -version = "0.8.3" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d746fc8a0dab19ccea7ff73ad535854e90ddb3b4b8cdce953dd5cd0b2e7bd22" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ - "antidote", - "log 0.4.11", + "log 0.4.17", + "parking_lot 0.12.1", "scheduled-thread-pool", ] @@ -1515,7 +1704,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] @@ -1528,7 +1717,7 @@ dependencies = [ "fuchsia-cprng", "libc", "rand_core 0.3.1", - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] @@ -1537,17 +1726,28 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "autocfg 0.1.2", + "autocfg 0.1.8", "libc", - "rand_chacha", - "rand_core 0.4.0", + "rand_chacha 0.1.1", + "rand_core 0.4.2", "rand_hc", "rand_isaac", "rand_jitter", "rand_os", "rand_pcg", "rand_xorshift", - "winapi 0.3.6", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", ] [[package]] @@ -1556,24 +1756,43 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" dependencies = [ - "autocfg 0.1.2", + "autocfg 0.1.8", "rand_core 0.3.1", ] [[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] name = "rand_core" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" dependencies = [ - "rand_core 0.4.0", + "rand_core 0.4.2", ] [[package]] name = "rand_core" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] name = "rand_hc" @@ -1595,37 +1814,37 @@ dependencies = [ [[package]] name = "rand_jitter" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080723c6145e37503a2224f801f252e14ac5531cb450f4502698542d188cb3c0" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" dependencies = [ "libc", - "rand_core 0.4.0", - "winapi 0.3.6", + "rand_core 0.4.2", + "winapi 0.3.9", ] [[package]] name = "rand_os" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c690732391ae0abafced5015ffb53656abfaec61b342290e5eb56b286a679d" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ "cloudabi", "fuchsia-cprng", "libc", - "rand_core 0.4.0", + "rand_core 0.4.2", "rdrand", - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] name = "rand_pcg" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086bd09a33c7044e56bb44d5bdde5a60e7f119a9e95b0775f545de759a32fe05" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "rand_core 0.3.1", - "rustc_version", + "autocfg 0.1.8", + "rand_core 0.4.2", ] [[package]] @@ -1648,17 +1867,28 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.51" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] -name = "redox_termios" -version = "0.1.1" +name = "redox_syscall" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "redox_syscall", + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", ] [[package]] @@ -1667,8 +1897,8 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" dependencies = [ - "aho-corasick 0.6.9", - "memchr 2.3.4", + "aho-corasick 0.6.10", + "memchr 2.5.0", "regex-syntax 0.5.6", "thread_local 0.3.6", "utf8-ranges", @@ -1676,14 +1906,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ - "aho-corasick 0.7.15", - "memchr 2.3.4", - "regex-syntax 0.6.21", - "thread_local 1.0.1", + "aho-corasick 0.7.19", + "memchr 2.5.0", + "regex-syntax 0.6.27", ] [[package]] @@ -1697,17 +1926,17 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] @@ -1724,29 +1953,29 @@ dependencies = [ "flate2", "futures", "http", - "hyper 0.12.23", - "log 0.4.11", - "mime 0.3.13", - "mime_guess 2.0.3", - "serde 1.0.117", - "serde_json 1.0.38", + "hyper 0.12.36", + "log 0.4.17", + "mime 0.3.16", + "mime_guess 2.0.4", + "serde", + "serde_json", "serde_urlencoded", - "time", + "time 0.1.44", "tokio", "tokio-executor", "tokio-io", "tokio-threadpool", "tokio-timer", "url 1.7.2", - "uuid 0.7.2", + "uuid 0.7.4", "winreg", ] [[package]] name = "route-recognizer" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" +checksum = "ea509065eb0b3c446acdd0102f0d46567dc30902dc0be91d6552035d92b0f4f8" [[package]] name = "router" @@ -1754,16 +1983,16 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc63b6f3b8895b0d04e816b2b1aa58fdba2d5acca3cbb8f0ab8e017347d57397" dependencies = [ - "iron 0.6.0", + "iron 0.6.1", "route-recognizer", "url 1.7.2", ] [[package]] name = "rustc-demangle" -version = "0.1.13" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adacaae16d02b6ec37fdc7acfcddf365978de76d1983d3ee22afc260e1ca9619" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustc-serialize" @@ -1781,10 +2010,16 @@ dependencies = [ ] [[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + +[[package]] name = "ryu" -version = "0.2.7" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "safemem" @@ -1794,17 +2029,26 @@ checksum = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" [[package]] name = "safemem" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] [[package]] name = "scheduled-thread-pool" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2ff3fc5223829be817806c6441279c676e454cc7da608faf03b0ccc09d3889" +checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" dependencies = [ - "antidote", + "parking_lot 0.12.1", ] [[package]] @@ -1814,6 +2058,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" [[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] name = "semver" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1839,9 +2089,9 @@ dependencies = [ "httpdate", "im", "lazy_static 1.4.0", - "log 0.4.11", + "log 0.4.17", "rand 0.6.5", - "regex 1.4.2", + "regex 1.6.0", "reqwest", "rustc_version", "sentry-types", @@ -1854,66 +2104,34 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23e3d9c8c6e4a1523f24df6753c4088bfe16c44a73c8881c1d23c70f28ae280" dependencies = [ - "chrono 0.4.6", + "chrono 0.4.22", "debugid", "failure", - "serde 1.0.117", - "serde_json 1.0.38", + "serde", + "serde_json", "url 1.7.2", "url_serde", - "uuid 0.7.2", + "uuid 0.7.4", ] [[package]] name = "serde" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" - -[[package]] -name = "serde" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" - -[[package]] -name = "serde" -version = "1.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" -dependencies = [ - "serde_derive 1.0.117", -] - -[[package]] -name = "serde_codegen_internals" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc888bd283bd2420b16ad0d860e35ad8acb21941180a83a189bb2046f9d00400" -dependencies = [ - "syn 0.11.11", -] - -[[package]] -name = "serde_derive" -version = "0.9.15" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978fd866f4d4872084a81ccc35e275158351d3b9fe620074e7d7504b816b74ba" +checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" dependencies = [ - "quote 0.3.15", - "serde_codegen_internals", - "syn 0.11.11", + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "syn 1.0.48", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1922,49 +2140,52 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142" dependencies = [ - "serde 1.0.117", + "serde", ] [[package]] name = "serde_json" -version = "0.9.10" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" +checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" dependencies = [ - "dtoa", - "itoa 0.3.4", - "num-traits 0.1.43", - "serde 0.9.15", -] - -[[package]] -name = "serde_json" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27dce848e7467aa0e2fcaf0a413641499c0b745452aaca1194d24dedde9e13c9" -dependencies = [ - "itoa 0.4.3", + "itoa 1.0.3", "ryu", - "serde 1.0.117", + "serde", ] [[package]] name = "serde_urlencoded" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48f9f99cd749a2de71d29da5f948de7f2764cc5a9d7f3c97e3514d4ee6eabf2" +checksum = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" dependencies = [ "dtoa", - "itoa 0.4.3", - "serde 1.0.117", + "itoa 0.4.8", + "serde", "url 1.7.2", ] [[package]] name = "sha1" -version = "0.6.0" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "signature" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +checksum = "deb766570a2825fa972bceff0d195727876a9cdf2460ab2e52d455dc2de47fd9" [[package]] name = "siphasher" @@ -1983,100 +2204,113 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg 1.1.0", +] [[package]] name = "slog" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e1a2eec401952cd7b12a84ea120e2d57281329940c3f93c2bf04f462539508e" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" [[package]] name = "slog-async" -version = "2.3.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e544d16c6b230d84c866662fe55e31aacfca6ae71e6fc49ae9a311cb379bfc2f" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" dependencies = [ + "crossbeam-channel 0.5.6", "slog", "take_mut", - "thread_local 0.3.6", + "thread_local 1.1.4", ] [[package]] name = "slog-scope" -version = "4.1.1" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60c04b4726fa04595ccf2c2dad7bcd15474242c4c5e109a8a376e8a2c9b1539a" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" dependencies = [ - "crossbeam 0.6.0", + "arc-swap", "lazy_static 1.4.0", "slog", ] [[package]] name = "slog-stdlog" -version = "3.0.2" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac42f8254ae996cc7d640f9410d3b048dcdf8887a10df4d5d4c44966de24c4a8" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ - "crossbeam 0.2.12", - "log 0.3.9", + "log 0.4.17", "slog", "slog-scope", ] [[package]] name = "slog-term" -version = "2.4.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5951a808c40f419922ee014c15b6ae1cd34d963538b57d8a4778b9ca3fff1e0b" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" dependencies = [ - "chrono 0.4.6", - "isatty", + "atty", "slog", "term", - "thread_local 0.3.6", + "thread_local 1.1.4", + "time 0.3.14", ] [[package]] name = "smallvec" -version = "0.6.8" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88aea073965ab29f6edb5493faf96ad662fb18aa9eeb186a3b7057951605ed15" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" dependencies = [ - "unreachable", + "maybe-uninit", ] [[package]] +name = "smallvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" + +[[package]] name = "sodiumoxide" -version = "0.0.13" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa64a17d98ec77bc7251c59a486e555b3813e32fb53ed608880f82e24ef6bd0" +checksum = "e26be3acb6c2d9a7aac28482586a7856436af4cfe7100031d219de2d2ecb0028" dependencies = [ + "ed25519", "libc", "libsodium-sys", - "serde 0.8.23", + "serde", ] [[package]] name = "stable_deref_trait" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "string" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b639411d0b9c738748b5397d5ceba08e648f4f1992231aa859af1a017f31f60b" +checksum = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" +dependencies = [ + "bytes", +] [[package]] name = "strsim" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "swagger" @@ -2085,65 +2319,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190ef0c6327759d0beb76d969b236fa3cc42469f9e107f626bbcc152727b4d12" dependencies = [ "base64 0.5.2", - "hyper 0.10.15", + "hyper 0.10.16", "iron 0.5.1", - "serde 1.0.117", - "serde_derive 1.0.117", - "serde_json 1.0.38", + "serde", + "serde_derive", + "serde_json", ] [[package]] name = "syn" -version = "0.11.11" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" dependencies = [ - "quote 0.3.15", - "synom", - "unicode-xid 0.0.4", -] - -[[package]] -name = "syn" -version = "0.15.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92e629aa1d9c827b2bb8297046c1ccffc57c99b947a680d3ccff1f136a3bee9" -dependencies = [ - "proc-macro2 0.4.27", - "quote 0.6.11", - "unicode-xid 0.1.0", -] - -[[package]] -name = "syn" -version = "1.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" -dependencies = [ - "proc-macro2 1.0.24", - "quote 1.0.7", - "unicode-xid 0.2.1", -] - -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -dependencies = [ - "unicode-xid 0.0.4", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] name = "synstructure" -version = "0.10.1" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 0.4.27", - "quote 0.6.11", - "syn 0.15.26", - "unicode-xid 0.1.0", + "proc-macro2", + "quote", + "syn", + "unicode-xid", ] [[package]] @@ -2164,41 +2367,51 @@ dependencies = [ [[package]] name = "term" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b677dd1e8214ea1ef4297f85dbcbed8e8cdddb561040cc998ca2551c37561" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ - "byteorder", - "winapi 0.3.6", + "dirs-next", + "rustversion", + "winapi 0.3.9", ] [[package]] name = "termcolor" -version = "1.0.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ - "wincolor", + "winapi-util", ] [[package]] -name = "termion" -version = "1.5.1" +name = "textwrap" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "libc", - "redox_syscall", - "redox_termios", + "unicode-width", ] [[package]] -name = "textwrap" -version = "0.10.0" +name = "thiserror" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" +checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" dependencies = [ - "unicode-width", + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2212,25 +2425,43 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ - "lazy_static 1.4.0", + "once_cell", ] [[package]] name = "time" -version = "0.1.42" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", - "redox_syscall", - "winapi 0.3.6", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", ] [[package]] +name = "time" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" +dependencies = [ + "itoa 1.0.3", + "libc", + "num_threads", + "time-macros", +] + +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + +[[package]] name = "tiny_http" version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2245,10 +2476,25 @@ dependencies = [ ] [[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] name = "tokio" -version = "0.1.15" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0500b88064f08bebddd0c0bed39e19f5c567a5f30975bee52b0c0d3e2eeb38c" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes", "futures", @@ -2264,10 +2510,21 @@ dependencies = [ ] [[package]] +name = "tokio-buf" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" +dependencies = [ + "bytes", + "either", + "futures", +] + +[[package]] name = "tokio-current-thread" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331c8acc267855ec06eb0c94618dcbbfea45bed2d20b77252940095273fb58f6" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" dependencies = [ "futures", "tokio-executor", @@ -2275,48 +2532,59 @@ dependencies = [ [[package]] name = "tokio-executor" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30c6dbf2d1ad1de300b393910e8a3aa272b724a400b6531da03eed99e329fbf0" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils 0.6.5", + "crossbeam-utils 0.7.2", "futures", ] [[package]] name = "tokio-io" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53aeb9d3f5ccf2ebb29e19788f96987fa1355f8fe45ea193928eaaaf3ae820f" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes", "futures", - "log 0.4.11", + "log 0.4.17", ] [[package]] name = "tokio-reactor" -version = "0.1.8" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afbcdb0f0d2a1e4c440af82d7bbf0bf91a8a8c0575bcd20c05d15be7e9d3a02f" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils 0.6.5", + "crossbeam-utils 0.7.2", "futures", "lazy_static 1.4.0", - "log 0.4.11", + "log 0.4.17", "mio", "num_cpus", - "parking_lot 0.7.1", + "parking_lot 0.9.0", "slab", "tokio-executor", "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures", ] [[package]] name = "tokio-tcp" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" dependencies = [ "bytes", "futures", @@ -2328,29 +2596,28 @@ dependencies = [ [[package]] name = "tokio-threadpool" -version = "0.1.11" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fd86cb15547d02daa2b21aadaf4e37dee3368df38a526178a5afa3c034d2fb" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ - "crossbeam 0.6.0", - "crossbeam-channel 0.3.8", "crossbeam-deque", - "crossbeam-utils 0.6.5", + "crossbeam-queue", + "crossbeam-utils 0.7.2", "futures", - "log 0.4.11", + "lazy_static 1.4.0", + "log 0.4.17", "num_cpus", - "rand 0.6.5", "slab", "tokio-executor", ] [[package]] name = "tokio-timer" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21c04a314a1f69f73c0227beba6250e06cdc1e9a62e7eff912bf54a59b6d1b94" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils 0.6.5", + "crossbeam-utils 0.7.2", "futures", "slab", "tokio-executor", @@ -2364,9 +2631,9 @@ checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" [[package]] name = "try-lock" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "try_from" @@ -2374,7 +2641,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 0.1.10", ] [[package]] @@ -2383,7 +2650,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" dependencies = [ - "memchr 2.3.4", + "memchr 2.5.0", ] [[package]] @@ -2403,15 +2670,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.10.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ucd-util" -version = "0.1.3" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" +checksum = "65bfcbf611b122f2c10eb1bb6172fbc4c2e25df9970330e4d75ce2b5201c9bfc" [[package]] name = "unicase" @@ -2428,59 +2695,41 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-normalization" -version = "0.1.8" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ - "smallvec", + "tinyvec", ] [[package]] name = "unicode-width" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" - -[[package]] -name = "unicode-xid" -version = "0.0.4" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - -[[package]] -name = "unicode-xid" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" - -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unsafe-any" @@ -2515,14 +2764,13 @@ dependencies = [ [[package]] name = "url" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna 0.2.0", - "matches", - "percent-encoding 2.1.0", + "idna 0.3.0", + "percent-encoding 2.2.0", ] [[package]] @@ -2531,7 +2779,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74e7d099f1ee52f823d4bdd60c93c3602043c728f5db3b97bdb548467f7bddea" dependencies = [ - "serde 1.0.117", + "serde", "url 1.7.2", ] @@ -2542,16 +2790,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a52f50139118b60ae91af08bf15ed158817d34b91b9d24c11ffbe21195d33e3" dependencies = [ "bodyparser", - "iron 0.6.0", + "iron 0.6.1", "plugin", "url 1.7.2", ] [[package]] name = "utf8-ranges" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" [[package]] name = "uuid" @@ -2570,7 +2818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" dependencies = [ "rand 0.3.23", - "serde 1.0.117", + "serde", ] [[package]] @@ -2579,30 +2827,30 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363" dependencies = [ - "cfg-if 0.1.6", + "cfg-if 0.1.10", ] [[package]] name = "uuid" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0238db0c5b605dd1cf51de0f21766f97fba2645897024461d6a00c036819a768" +checksum = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a" dependencies = [ "rand 0.6.5", - "serde 1.0.117", + "serde", ] [[package]] name = "vcpkg" -version = "0.2.6" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -2612,28 +2860,99 @@ checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "void" -version = "1.0.2" +name = "walkdir" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi 0.3.9", + "winapi-util", +] [[package]] name = "want" -version = "0.0.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" +checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ "futures", - "log 0.4.11", + "log 0.4.17", "try-lock", ] [[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log 0.4.17", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] name = "winapi" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2641,9 +2960,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -2663,11 +2982,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] @@ -2677,22 +2996,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "wincolor" -version = "1.0.1" +name = "windows-sys" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "winapi 0.3.6", - "winapi-util", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", ] [[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] name = "winreg" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf67b95d0b1bf421c4f11048d63110ca3719977169eec86396b614c8942b6e0" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" dependencies = [ - "winapi 0.3.6", + "winapi 0.3.9", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b521316e..37aee6b4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -23,7 +23,7 @@ data-encoding = "2.1" regex = "1" lazy_static = "1" sha1 = { version = "0.6", features = ["std"] } -macaroon = { git = "https://github.com/bnewbold/libmacaroon-rs", branch = "bnewbold-broken" } +macaroon = { git = "https://github.com/bnewbold/macaroon", branch = "bnewbold-legacy-hmac-fixed-branch" } rand = "*" failure = "*" diff --git a/rust/Makefile b/rust/Makefile index e222cf07..a66ae47b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -9,9 +9,25 @@ help: ## Print info about all commands @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[01;32m%-20s\033[0m %s\n", $$1, $$2}' .PHONY: test -test: ## Run all tests and lints +test: ## Run all tests cargo test -- --test-threads 1 +.PHONY: lint +lint: ## Run syntax/style checks + cargo clippy -- --no-deps + +.PHONY: fmt +fmt: ## Run syntax re-formatting + cargo fmt + +.PHONY: build +build: ## Build + cargo build + +.PHONY: build-release +build-release: ## Build for release + cargo build --release + .PHONY: serve serve: ## Run web service locally, with reloading #./target/debug/fatcatd diff --git a/rust/src/auth.rs b/rust/src/auth.rs index b88a72b0..3a6271e5 100644 --- a/rust/src/auth.rs +++ b/rust/src/auth.rs @@ -5,7 +5,7 @@ //! role-based authentication (RBAC). use data_encoding::BASE64; -use macaroon::{Format, Macaroon, Verifier}; +use macaroon::{ByteString, Caveat, Format, Macaroon, MacaroonError, MacaroonKey, Verifier}; use std::fmt; use swagger::auth::{AuthData, Authorization, Scopes}; @@ -23,10 +23,10 @@ use std::collections::HashMap; use std::env; use std::str::FromStr; -// 32 bytes max (!) -static DUMMY_KEY: &[u8] = b"dummy-key-a-one-two-three-a-la"; +// 32 bytes exactly +static DUMMY_KEY_BYTES: &[u8; 32] = b"dummy-key-a-one-two-three-a-la!!"; -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FatcatRole { Public, Editor, @@ -178,8 +178,23 @@ impl iron::middleware::BeforeMiddleware for MacaroonAuthMiddleware { pub struct AuthConfectionary { pub location: String, pub identifier: String, - pub key: Vec<u8>, - pub root_keys: HashMap<String, Vec<u8>>, + pub key: MacaroonKey, + pub root_keys: HashMap<String, MacaroonKey>, +} + +fn parse_macaroon_key(key_base64: &str) -> Result<MacaroonKey> { + // instead of creating a [u8; 32], we decode into an arbitrary Vec (after checking the input + // length first), because the MacaroonKey 'From' trait is implemented differently for [u8] and + // [u8; 32] (sigh). + if key_base64.len() != 44 { + bail!("bad base64-padded-encoded key for macaroons"); + } + let key_bytes = BASE64 + .decode(key_base64.as_bytes()) + .expect("base64 key decode"); + let bytes_ref: &[u8] = key_bytes.as_ref(); + let key = MacaroonKey::from(bytes_ref); + Ok(key) } impl AuthConfectionary { @@ -188,10 +203,10 @@ impl AuthConfectionary { identifier: String, key_base64: &str, ) -> Result<AuthConfectionary> { - macaroon::initialize().unwrap(); - let key = BASE64.decode(key_base64.as_bytes())?; + macaroon::initialize().expect("initializing macaroon library"); + let key = parse_macaroon_key(key_base64)?; let mut root_keys = HashMap::new(); - root_keys.insert(identifier.clone(), key.clone()); + root_keys.insert(identifier.clone(), key); Ok(AuthConfectionary { location, identifier, @@ -204,13 +219,13 @@ impl AuthConfectionary { AuthConfectionary::new( "test.fatcat.wiki".to_string(), "dummy".to_string(), - &BASE64.encode(DUMMY_KEY), + &BASE64.encode(DUMMY_KEY_BYTES), ) - .unwrap() + .expect("creating dummy AuthConfectionary") } pub fn add_keypair(&mut self, identifier: String, key_base64: &str) -> Result<()> { - let key = BASE64.decode(key_base64.as_bytes())?; + let key = parse_macaroon_key(key_base64)?; self.root_keys.insert(identifier, key); Ok(()) } @@ -220,23 +235,31 @@ impl AuthConfectionary { editor_id: FatcatId, duration: Option<chrono::Duration>, ) -> Result<String> { - let mut mac = Macaroon::create(&self.location, &self.key, &self.identifier) - .expect("Macaroon creation"); - mac.add_first_party_caveat(&format!("editor_id = {}", editor_id.to_string())); + let mut mac = Macaroon::create( + Some(self.location.clone()), + &self.key, + self.identifier.clone().into(), + ) + .expect("Macaroon creation"); + mac.add_first_party_caveat(format!("editor_id = {}", editor_id).into()); let now_utc = Utc::now(); let now = now_utc.to_rfc3339_opts(SecondsFormat::Secs, true); - mac.add_first_party_caveat(&format!("time > {}", now)); + mac.add_first_party_caveat(format!("time > {}", now).into()); if let Some(duration) = duration { let expires = now_utc + duration; - mac.add_first_party_caveat(&format!( - "time < {}", - &expires.to_rfc3339_opts(SecondsFormat::Secs, true) - )); + mac.add_first_party_caveat( + format!( + "time < {}", + &expires.to_rfc3339_opts(SecondsFormat::Secs, true) + ) + .into(), + ); }; let raw = mac.serialize(Format::V2).expect("macaroon serialization"); Ok(BASE64.encode(&raw)) } + /// Takes a macaroon as a base64-encoded string, deserializes it pub fn parse_macaroon_token( &self, conn: &DbConn, @@ -255,23 +278,19 @@ impl AuthConfectionary { .into()); } }; - let mac = match mac.validate() { - Ok(m) => m, - Err(e) => { - // TODO: should be "chaining" here - return Err(FatcatError::InvalidCredentials(format!( - "macaroon validate error: {:?}", - e - )) - .into()); - } - }; - let mut verifier = Verifier::new(); + let mut verifier = Verifier::default(); let mut editor_id: Option<FatcatId> = None; for caveat in mac.first_party_caveats() { - if caveat.predicate().starts_with("editor_id = ") { - editor_id = Some(FatcatId::from_str(caveat.predicate().get(12..).unwrap())?); - break; + if let Caveat::FirstParty(fp) = caveat { + let predicate_str = String::from_utf8(fp.predicate().as_ref().to_vec())?; + if predicate_str.starts_with("editor_id = ") { + editor_id = Some(FatcatId::from_str( + predicate_str + .get(12..) + .expect("parsing editor_id from macaroon"), + )?); + break; + } } } let editor_id = match editor_id { @@ -283,25 +302,28 @@ impl AuthConfectionary { .into()); } }; - verifier.satisfy_exact(&format!("editor_id = {}", editor_id.to_string())); + verifier.satisfy_exact(format!("editor_id = {}", editor_id).into()); if let Some(endpoint) = endpoint { // API endpoint - verifier.satisfy_exact(&format!("endpoint = {}", endpoint)); + verifier.satisfy_exact(format!("endpoint = {}", endpoint).into()); } let mut created: Option<DateTime<Utc>> = None; for caveat in mac.first_party_caveats() { - if caveat.predicate().starts_with("time > ") { - let ts: chrono::ParseResult<DateTime<Utc>> = - DateTime::parse_from_rfc3339(caveat.predicate().get(7..).unwrap()) - .map(|x| x.with_timezone(&Utc)); - if let Ok(ts) = ts { - created = Some(ts); - break; - } else { - info!( - "couldn't parse macaroon time constraint: {}", - caveat.predicate() - ); + if let Caveat::FirstParty(fp) = caveat { + let predicate_str = String::from_utf8(fp.predicate().as_ref().to_vec())?; + if predicate_str.starts_with("time > ") { + let ts: chrono::ParseResult<DateTime<Utc>> = DateTime::parse_from_rfc3339( + predicate_str + .get(7..) + .expect("parsing timestamp from macaroon"), + ) + .map(|x| x.with_timezone(&Utc)); + if let Ok(ts) = ts { + created = Some(ts); + break; + } else { + info!("couldn't parse macaroon time constraint: {}", predicate_str); + } } } } @@ -314,23 +336,25 @@ impl AuthConfectionary { .into()); } }; - verifier.satisfy_exact(&format!( - "time > {}", - created.to_rfc3339_opts(SecondsFormat::Secs, true) - )); + verifier.satisfy_exact( + format!( + "time > {}", + created.to_rfc3339_opts(SecondsFormat::Secs, true) + ) + .into(), + ); // not finding the editor_id is an auth issue, not a 404 - let editor: EditorRow = - match Editor::db_get(conn, editor_id).map_err(|e| FatcatError::from(e)) { - Ok(ed) => ed, - Err(FatcatError::NotFound(_, _)) => { - return Err(FatcatError::InvalidCredentials(format!( - "editor_id not found: {}", - editor_id - )) - .into()); - } - other_db_err => other_db_err?, - }; + let editor: EditorRow = match Editor::db_get(conn, editor_id).map_err(FatcatError::from) { + Ok(ed) => ed, + Err(FatcatError::NotFound(_, _)) => { + return Err(FatcatError::InvalidCredentials(format!( + "editor_id not found: {}", + editor_id + )) + .into()); + } + other_db_err => other_db_err?, + }; let auth_epoch = DateTime::<Utc>::from_utc(editor.auth_epoch, Utc); // allow a second of wiggle room for precision and, eg, tests if created < (auth_epoch - chrono::Duration::seconds(1)) { @@ -340,39 +364,48 @@ impl AuthConfectionary { ) .into()); } - verifier.satisfy_general(|p: &str| -> bool { - // not expired (based on time) - if p.starts_with("time < ") { - let expires: chrono::ParseResult<DateTime<Utc>> = - DateTime::parse_from_rfc3339(p.get(7..).unwrap()) - .map(|x| x.with_timezone(&Utc)); - if let Ok(when) = expires { - //info!("checking time constraint: {} < {}", Utc::now(), when); - Utc::now() < when + verifier.satisfy_general(|p_bstr: &ByteString| -> bool { + if let Ok(p) = String::from_utf8(p_bstr.as_ref().to_vec()) { + // not expired (based on time) + if p.starts_with("time < ") { + let expires: chrono::ParseResult<DateTime<Utc>> = DateTime::parse_from_rfc3339( + p.get(7..).expect("parsing datetime from macaroon"), + ) + .map(|x| x.with_timezone(&Utc)); + if let Ok(when) = expires { + //info!("checking time constraint: {} < {}", Utc::now(), when); + Utc::now() < when + } else { + info!("couldn't parse macaroon time constraint: {}", p); + false + } } else { - info!("couldn't parse macaroon time constraint: {}", p); false } } else { false } }); - let verify_key = match self.root_keys.get(mac.identifier()) { + let verify_key = match self + .root_keys + .get(std::str::from_utf8(mac.identifier().as_ref())?) + { Some(key) => key, None => { return Err(FatcatError::InvalidCredentials(format!( "no valid auth signing key for identifier: {}", - mac.identifier() + std::str::from_utf8(mac.identifier().as_ref())? )) .into()); } }; - match mac.verify(verify_key, &mut verifier) { - Ok(true) => (), - Ok(false) => { - return Err(FatcatError::InvalidCredentials( - "auth token (macaroon) not valid (signature and/or caveats failed)".to_string(), - ) + match verifier.verify(&mac, verify_key, Default::default()) { + Ok(()) => (), + Err(MacaroonError::InvalidMacaroon(em)) => { + return Err(FatcatError::InvalidCredentials(format!( + "auth token (macaroon) not valid (signature and/or caveats failed): {}", + em + )) .into()); } Err(e) => { @@ -448,9 +481,15 @@ impl AuthConfectionary { let now = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); println!("current time: {}", now); println!("domain (location): {:?}", mac.location()); - println!("signing key name (identifier): {}", mac.identifier()); + println!( + "signing key name (identifier): {}", + std::str::from_utf8(mac.identifier().as_ref())? + ); for caveat in mac.first_party_caveats() { - println!("caveat: {}", caveat.predicate()); + if let Caveat::FirstParty(fp) = caveat { + let predicate_str = String::from_utf8(fp.predicate().as_ref().to_vec())?; + println!("caveat: {}", predicate_str); + } } println!("verify: {:?}", self.parse_macaroon_token(conn, token, None)); Ok(()) @@ -488,7 +527,7 @@ pub fn print_editors(conn: &DbConn) -> Result<()> { for e in all_editors { println!( "{}\t{}/{}/{}\t{}\t{}\t{:?}", - FatcatId::from_uuid(&e.id).to_string(), + FatcatId::from_uuid(&e.id), e.is_superuser, e.is_admin, e.is_bot, @@ -519,3 +558,25 @@ pub fn env_confectionary() -> Result<AuthConfectionary> { }; Ok(confectionary) } + +#[test] +fn test_macaroon_keys() { + assert!(parse_macaroon_key("blah").is_err()); + + let key_bytes: [u8; 32] = [ + 231, 158, 121, 231, 158, 121, 231, 158, 121, 231, 158, 121, 231, 158, 121, 231, 158, 121, + 231, 158, 121, 231, 158, 121, 231, 158, 121, 231, 158, 121, 198, 107, + ]; + + // old way of parsing keys + let old_key = BASE64 + .decode(b"5555555555555555555555555555555555555555xms=") + .unwrap(); + assert_eq!(old_key.len(), 32); + assert_eq!(old_key, key_bytes); + let old_macaroon_key: MacaroonKey = old_key.as_slice().into(); + + // new (2022) way of parsing keys + let key = parse_macaroon_key("5555555555555555555555555555555555555555xms=").unwrap(); + assert_eq!(old_macaroon_key, key); +} diff --git a/rust/src/bin/fatcat-auth.rs b/rust/src/bin/fatcat-auth.rs index 0b5c05b0..c351848e 100644 --- a/rust/src/bin/fatcat-auth.rs +++ b/rust/src/bin/fatcat-auth.rs @@ -89,7 +89,7 @@ fn main() -> Result<()> { }; let editor_row = editor.db_create(&db_conn)?; //println!("{:?}", editor); - println!("{}", FatcatId::from_uuid(&editor_row.id).to_string()); + println!("{}", FatcatId::from_uuid(&editor_row.id)); } ("create-token", Some(subm)) => { let editor_id = FatcatId::from_str(subm.value_of("editor-id").unwrap())?; diff --git a/rust/src/bin/fatcat-doctor.rs b/rust/src/bin/fatcat-doctor.rs new file mode 100644 index 00000000..7255a22f --- /dev/null +++ b/rust/src/bin/fatcat-doctor.rs @@ -0,0 +1,96 @@ +//! Database cleanup tool + +use clap::{value_t_or_exit, App, SubCommand}; + +use fatcat::database_models::*; +use fatcat::database_schema::*; +use fatcat::errors::Result; +use fatcat::identifiers::FatcatId; +use fatcat::server; +use fatcat::server::DbConn; +use std::process; +use std::str::FromStr; + +use diesel::prelude::*; + +fn backfill_changelog_gap(conn: &DbConn, last_good: i64, max_index: i64) -> Result<()> { + // sanity check arguments against database + assert!(last_good > 0); + assert!(max_index > 0); + assert!(last_good < max_index); + let highest_row: ChangelogRow = changelog::table.order(changelog::id.desc()).first(conn)?; + assert!(highest_row.id >= max_index); + + // default values + // 'root' editor_id is aaaaaaaaaaaabkvkaaaaaaaaae + // 'admin' editor_id is aaaaaaaaaaaabkvkaaaaaaaaai + let editor_id = FatcatId::from_str("aaaaaaaaaaaabkvkaaaaaaaaai").unwrap(); + let description = "Backfill of missing changelog entries due to database id gap"; + + // fetch the last entry before the gap, to re-use the timestamp + let existing_row: ChangelogRow = changelog::table.find(last_good).first(conn)?; + + for index in last_good + 1..max_index + 1 { + // ensure this index is actually a gap + let count: i64 = changelog::table + .filter(changelog::id.eq(index)) + .count() + .get_result(conn)?; + if count != 0 { + println!("Found existing changelog: {}", index); + return Ok(()); + } + + // create dummy empty editgroup, then add a changelog entry + let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) + .values(( + editgroup::editor_id.eq(editor_id.to_uuid()), + editgroup::created.eq(existing_row.timestamp), + editgroup::is_accepted.eq(true), + editgroup::description.eq(Some(description)), + )) + .get_result(conn)?; + let _entry_row: ChangelogRow = diesel::insert_into(changelog::table) + .values(( + changelog::id.eq(index), + changelog::editgroup_id.eq(eg_row.id), + changelog::timestamp.eq(existing_row.timestamp), + )) + .get_result(conn)?; + println!("Inserted changelog: {}", index); + } + Ok(()) +} + +fn main() -> Result<()> { + let m = App::new("fatcat-doctor") + .version(env!("CARGO_PKG_VERSION")) + .author("Bryan Newbold <bnewbold@archive.org>") + .about("Database cleanup / fixup tool") + .subcommand( + SubCommand::with_name("backfill-changelog-gap") + .about("Inserts dummy changelog entries and editgroups for gap") + .args_from_usage( + "<start> 'changelog index of entry just before gap' + <end> 'highest changelog index to backfill'", + ), + ) + .get_matches(); + + let db_conn = server::database_worker_pool()? + .get() + .expect("database pool"); + match m.subcommand() { + ("backfill-changelog-gap", Some(subm)) => { + let last_good: i64 = value_t_or_exit!(subm.value_of("start"), i64); + let max_index: i64 = value_t_or_exit!(subm.value_of("end"), i64); + backfill_changelog_gap(&db_conn, last_good, max_index)?; + } + _ => { + println!("Missing or unimplemented command!"); + println!("{}", m.usage()); + process::exit(-1); + } + } + Ok(()) +} diff --git a/rust/src/bin/fatcat-export.rs b/rust/src/bin/fatcat-export.rs index 7d671b9a..9ac977aa 100644 --- a/rust/src/bin/fatcat-export.rs +++ b/rust/src/bin/fatcat-export.rs @@ -28,7 +28,7 @@ use std::thread; const CHANNEL_BUFFER_LEN: usize = 200; arg_enum! { - #[derive(PartialEq, Debug, Clone, Copy)] + #[derive(PartialEq, Debug, Clone, Copy, Eq)] pub enum ExportEntityType { Creator, Container, @@ -163,7 +163,7 @@ fn parse_line(s: &str) -> Result<IdentRow> { let group_id: Option<FatcatId> = if fields.len() == 4 { match fields[3].as_ref() { "" => None, - val => Some(FatcatId::from_uuid(&Uuid::from_str(&val)?)), + val => Some(FatcatId::from_uuid(&Uuid::from_str(val)?)), } } else if fields.len() == 3 { None @@ -174,11 +174,11 @@ fn parse_line(s: &str) -> Result<IdentRow> { ident_id: FatcatId::from_uuid(&Uuid::from_str(&fields[0])?), rev_id: match fields[1].as_ref() { "" => None, - val => Some(Uuid::from_str(&val)?), + val => Some(Uuid::from_str(val)?), }, redirect_id: match fields[2].as_ref() { "" => None, - val => Some(FatcatId::from_uuid(&Uuid::from_str(&val)?)), + val => Some(FatcatId::from_uuid(&Uuid::from_str(val)?)), }, group_id, }) @@ -331,7 +331,7 @@ pub fn do_export_batch( (Some(_), Some(_), false) => (), _ => { if row.group_id == None || row.group_id != last_group_id { - if batch.len() > 0 { + if !batch.is_empty() { row_sender.send(batch); batch = vec![]; } @@ -345,7 +345,7 @@ pub fn do_export_batch( info!("processed {} lines...", count); } } - if batch.len() > 0 { + if !batch.is_empty() { row_sender.send(batch); } drop(row_sender); @@ -385,7 +385,7 @@ fn main() -> Result<()> { None => std::cmp::min(1, num_cpus::get() / 2) as usize, }; let expand = match m.value_of("expand") { - Some(s) => Some(ExpandFlags::from_str(&s)?), + Some(s) => Some(ExpandFlags::from_str(s)?), None => None, }; let log_level = if m.is_present("quiet") { diff --git a/rust/src/bin/fatcatd.rs b/rust/src/bin/fatcatd.rs index 006154df..bfa41805 100644 --- a/rust/src/bin/fatcatd.rs +++ b/rust/src/bin/fatcatd.rs @@ -27,7 +27,7 @@ pub struct XClacksOverheadMiddleware; impl AfterMiddleware for XClacksOverheadMiddleware { fn after(&self, _req: &mut Request, mut res: Response) -> iron::IronResult<Response> { res.headers - .set(XClacksOverhead("GNU aaronsw, jpb".to_owned())); + .set(XClacksOverhead("GNU aaronsw, jpb, pde".to_owned())); Ok(res) } } @@ -48,7 +48,7 @@ fn main() -> Result<()> { let drain = slog_async::Async::new(drain).build().fuse(); let logger = Logger::root(drain, o!()); let _scope_guard = slog_scope::set_global_logger(logger.clone()); - let _log_guard = slog_stdlog::init().unwrap(); + slog_stdlog::init().unwrap(); let formatter = DefaultLogFormatter; // sentry exception handling diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs index 0427f9c8..b3b64126 100644 --- a/rust/src/database_models.rs +++ b/rust/src/database_models.rs @@ -13,7 +13,7 @@ use uuid::Uuid; // Ugh. I thought the whole point was to *not* do this, but: // https://github.com/diesel-rs/diesel/issues/1589 -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EntityState { WorkInProgress, Active(Uuid), @@ -552,7 +552,7 @@ pub struct RefsBlobRow { pub refs_json: serde_json::Value, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq)] /// This model is a stable representation of what goes in a RefsBlobRow `refs_json` field (an array /// of this model). We could rely on the `ReleaseRef` API spec model directly, but that would lock /// the database contents to the API spec rigidly; by defining this struct independently, we can @@ -684,8 +684,8 @@ impl EditgroupRow { Editgroup { editgroup_id: Some(uuid2fcid(&self.id)), editor_id: Some(uuid2fcid(&self.editor_id)), - editor: editor, - changelog_index: changelog_index, + editor, + changelog_index, created: Some(chrono::DateTime::from_utc(self.created, chrono::Utc)), submitted: self .submitted diff --git a/rust/src/endpoint_handlers.rs b/rust/src/endpoint_handlers.rs index 9a76652b..ed6f6641 100644 --- a/rust/src/endpoint_handlers.rs +++ b/rust/src/endpoint_handlers.rs @@ -171,7 +171,7 @@ impl Server { }; let mut entity = ContainerEntity::db_from_row(conn, rev, Some(ident), hide_flags)?; - entity.db_expand(&conn, expand_flags)?; + entity.db_expand(conn, expand_flags)?; Ok(entity) } @@ -210,7 +210,7 @@ impl Server { }; let mut entity = CreatorEntity::db_from_row(conn, rev, Some(ident), hide_flags)?; - entity.db_expand(&conn, expand_flags)?; + entity.db_expand(conn, expand_flags)?; Ok(entity) } @@ -280,7 +280,7 @@ impl Server { }; let mut entity = FileEntity::db_from_row(conn, rev, Some(ident), hide_flags)?; - entity.db_expand(&conn, expand_flags)?; + entity.db_expand(conn, expand_flags)?; Ok(entity) } @@ -705,7 +705,7 @@ impl Server { }; let mut entity = ReleaseEntity::db_from_row(conn, rev, Some(ident), hide_flags)?; - entity.db_expand(&conn, expand_flags)?; + entity.db_expand(conn, expand_flags)?; Ok(entity) } @@ -906,7 +906,7 @@ impl Server { username.truncate(24); let editor = Editor { editor_id: None, - username: username, + username, is_admin: Some(false), is_bot: Some(false), is_active: Some(true), diff --git a/rust/src/entity_crud.rs b/rust/src/entity_crud.rs index e8f63d0c..6dc9ff18 100644 --- a/rust/src/entity_crud.rs +++ b/rust/src/entity_crud.rs @@ -96,7 +96,7 @@ where fn db_insert_revs(conn: &DbConn, models: &[&Self]) -> Result<Vec<Uuid>>; } -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, Copy, PartialEq, Eq)] pub struct ExpandFlags { pub files: bool, pub filesets: bool, @@ -187,7 +187,7 @@ fn test_expand_flags() { ); } -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, Copy, PartialEq, Eq)] pub struct HideFlags { // release pub abstracts: bool, @@ -1195,7 +1195,7 @@ impl EntityCrud for FileEntity { None => (), Some(url_list) => { let these_url_rows: Vec<FileRevUrlNewRow> = url_list - .into_iter() + .iter() .map(|u| FileRevUrlNewRow { file_rev: *rev_id, rel: u.rel.clone(), @@ -1399,7 +1399,7 @@ impl EntityCrud for FilesetEntity { None => (), Some(file_list) => { let these_file_rows: Vec<FilesetRevFileNewRow> = file_list - .into_iter() + .iter() .map(|f| FilesetRevFileNewRow { fileset_rev: *rev_id, path_name: f.path.clone(), @@ -1419,7 +1419,7 @@ impl EntityCrud for FilesetEntity { None => (), Some(url_list) => { let these_url_rows: Vec<FilesetRevUrlNewRow> = url_list - .into_iter() + .iter() .map(|u| FilesetRevUrlNewRow { fileset_rev: *rev_id, rel: u.rel.clone(), @@ -1654,11 +1654,11 @@ impl EntityCrud for WebcaptureEntity { None => (), Some(cdx_list) => { let these_cdx_rows: Vec<WebcaptureRevCdxNewRow> = cdx_list - .into_iter() + .iter() .map(|c| WebcaptureRevCdxNewRow { webcapture_rev: *rev_id, surt: c.surt.clone(), - timestamp: c.timestamp.clone(), + timestamp: c.timestamp, url: c.url.clone(), mimetype: c.mimetype.clone(), status_code: c.status_code, @@ -1675,7 +1675,7 @@ impl EntityCrud for WebcaptureEntity { None => (), Some(url_list) => { let these_url_rows: Vec<WebcaptureRevUrlNewRow> = url_list - .into_iter() + .iter() .map(|u| WebcaptureRevUrlNewRow { webcapture_rev: *rev_id, rel: u.rel.clone(), @@ -1818,8 +1818,8 @@ impl EntityCrud for ReleaseEntity { Some(ident) => match &self.redirect { // If we're a redirect, then expand for the *target* identifier, not *our* // identifier. Tricky! - None => FatcatId::from_str(&ident)?, - Some(redir) => FatcatId::from_str(&redir)?, + None => FatcatId::from_str(ident)?, + Some(redir) => FatcatId::from_str(redir)?, }, }; self.files = Some(get_release_files(conn, ident, HideFlags::none())?); @@ -1828,8 +1828,8 @@ impl EntityCrud for ReleaseEntity { let ident = match &self.ident { None => bail!("Can't expand filesets on a non-concrete entity"), // redundant with above is_some() Some(ident) => match &self.redirect { - None => FatcatId::from_str(&ident)?, - Some(redir) => FatcatId::from_str(&redir)?, + None => FatcatId::from_str(ident)?, + Some(redir) => FatcatId::from_str(redir)?, }, }; self.filesets = Some(get_release_filesets(conn, ident, HideFlags::none())?); @@ -1838,8 +1838,8 @@ impl EntityCrud for ReleaseEntity { let ident = match &self.ident { None => bail!("Can't expand webcaptures on a non-concrete entity"), // redundant with above is_some() Some(ident) => match &self.redirect { - None => FatcatId::from_str(&ident)?, - Some(redir) => FatcatId::from_str(&redir)?, + None => FatcatId::from_str(ident)?, + Some(redir) => FatcatId::from_str(redir)?, }, }; self.webcaptures = Some(get_release_webcaptures(conn, ident, HideFlags::none())?); @@ -1848,7 +1848,7 @@ impl EntityCrud for ReleaseEntity { if let Some(ref cid) = self.container_id { self.container = Some(ContainerEntity::db_get( conn, - FatcatId::from_str(&cid)?, + FatcatId::from_str(cid)?, HideFlags::none(), )?); } @@ -1929,7 +1929,7 @@ impl EntityCrud for ReleaseEntity { model }) .collect(); - let model_refs: Vec<&Self> = models_with_work_ids.iter().map(|s| s).collect(); + let model_refs: Vec<&Self> = models_with_work_ids.iter().collect(); let models = model_refs.as_slice(); // The rest here is copy/pasta from the generic (how to avoid copypasta?) @@ -1952,7 +1952,7 @@ impl EntityCrud for ReleaseEntity { rev_ids .into_iter() .zip(ident_ids.into_iter()) - .zip(models.into_iter().map(|m| m.edit_extra.clone())) + .zip(models.iter().map(|m| m.edit_extra.clone())) .map(|((rev_id, ident_id), edit_extra)| Self::EditNewRow { editgroup_id: edit_context.editgroup_id.to_uuid(), rev_id: Some(rev_id), @@ -2101,7 +2101,7 @@ impl EntityCrud for ReleaseEntity { withdrawn_status: rev_row.withdrawn_status, withdrawn_date: rev_row.withdrawn_date, withdrawn_year: rev_row.withdrawn_year, - ext_ids: ext_ids, + ext_ids, volume: rev_row.volume, issue: rev_row.issue, pages: rev_row.pages, @@ -2460,7 +2460,7 @@ impl EntityCrud for ReleaseEntity { .collect(); abstract_rows.extend(new_abstracts); let new_release_abstract_rows: Vec<ReleaseRevAbstractNewRow> = abstract_list - .into_iter() + .iter() .map(|c| { Ok(ReleaseRevAbstractNewRow { release_rev: *rev_id, diff --git a/rust/src/errors.rs b/rust/src/errors.rs index ea0f9646..38429802 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -173,16 +173,16 @@ impl From<data_encoding::DecodeError> for FatcatError { impl From<failure::Error> for FatcatError { fn from(error: failure::Error) -> FatcatError { // TODO: I think it should be possible to match here? regardless, this is *super* janky - if let Some(_) = error.downcast_ref::<FatcatError>() { + if error.downcast_ref::<FatcatError>().is_some() { return error.downcast::<FatcatError>().unwrap(); } - if let Some(_) = error.downcast_ref::<std::fmt::Error>() { + if error.downcast_ref::<std::fmt::Error>().is_some() { return error.downcast::<std::fmt::Error>().unwrap().into(); } - if let Some(_) = error.downcast_ref::<diesel::result::Error>() { + if error.downcast_ref::<diesel::result::Error>().is_some() { return error.downcast::<diesel::result::Error>().unwrap().into(); } - if let Some(_) = error.downcast_ref::<uuid::ParseError>() { + if error.downcast_ref::<uuid::ParseError>().is_some() { return error.downcast::<uuid::ParseError>().unwrap().into(); } FatcatError::InternalError(error.to_string()) diff --git a/rust/src/identifiers.rs b/rust/src/identifiers.rs index e9baf7b8..56174be1 100644 --- a/rust/src/identifiers.rs +++ b/rust/src/identifiers.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use std::{convert, fmt}; use uuid::Uuid; -#[derive(Clone, Copy, PartialEq, Debug)] +#[derive(Clone, Copy, PartialEq, Debug, Eq)] pub struct FatcatId(Uuid); impl fmt::Display for FatcatId { @@ -51,7 +51,7 @@ impl FatcatId { /// Convert fatcat IDs (base32 strings) to UUID pub fn fcid2uuid(fcid: &str) -> Result<Uuid> { - if fcid.is_ascii() == false || fcid.len() != 26 { + if !fcid.is_ascii() || fcid.len() != 26 { return Err(FatcatError::InvalidFatcatId(fcid.to_string()).into()); } let mut raw = vec![0; 16]; diff --git a/rust/tests/test_auth.rs b/rust/tests/test_auth.rs index c684490f..021092e0 100644 --- a/rust/tests/test_auth.rs +++ b/rust/tests/test_auth.rs @@ -4,6 +4,26 @@ use fatcat::{auth, server}; use std::str::FromStr; #[test] +fn test_old_token() { + let server = server::create_test_server().unwrap(); + let conn = server.db_pool.get().expect("db_pool error"); + + let admin_dev_token = "AgEPZGV2LmZhdGNhdC53aWtpAhYyMDE5MDEwMS1kZXYtZHVtbXkta2V5AAImZWRpdG9yX2lkID0gYWFhYWFhYWFhYWFhYmt2a2FhYWFhYWFhYWkAAht0aW1lID4gMjAxOS0wNC0wNFQyMzozMjo0NloAAAYgrN3jjy0mgEqIydTFfsOLYSS55dz6Fh2d1CGMNQFLwcQ="; + let editor_id = FatcatId::from_str("aaaaaaaaaaaabkvkaaaaaaaaai").unwrap(); + + let c = AuthConfectionary::new( + "dev.fatcat.wiki".to_string(), + "20190101-dev-dummy-key".to_string(), + "5555555555555555555555555555555555555555xms=", + ) + .unwrap(); + let editor_row = c + .parse_macaroon_token(&conn, &admin_dev_token, None) + .unwrap(); + assert_eq!(editor_row.id, editor_id.to_uuid()); +} + +#[test] fn test_macaroons() { // Test everything we can without connecting to database @@ -40,6 +60,7 @@ fn test_auth_db() { .unwrap(); // verify token w/ expiration + assert!(c.parse_macaroon_token(&conn, &token, None).is_ok()); let editor_row = c.parse_macaroon_token(&conn, &token, None).unwrap(); assert_eq!(editor_row.id, editor_id.to_uuid()); @@ -47,6 +68,48 @@ fn test_auth_db() { auth::revoke_tokens(&conn, editor_id).unwrap(); // verification should fail - // TODO: one-second slop breaks this + // TODO: one-second slop breaks this; sleep for 1-2 seconds? //assert!(c.parse_macaroon_token(&conn, &token, None).is_err()); + + // bad macaroon (not base64) + assert!(c + .parse_macaroon_token(&conn, "some string with spaces", None) + .is_err()); + + // bad macaroon (wrong key used to sign) + let other_key = fatcat::auth::create_key(); + let c_other_key = AuthConfectionary::new( + "test.fatcat.wiki".to_string(), + "other-dummy".to_string(), + &other_key, + ) + .expect("creating dummy AuthConfectionary"); + let token_other_key = c_other_key.create_token(editor_id, None).unwrap(); + assert!(c_other_key + .parse_macaroon_token(&conn, &token_other_key, None) + .is_ok()); + assert!(c + .parse_macaroon_token(&conn, &token_other_key, None) + .is_err()); + assert!(c_other_key + .parse_macaroon_token(&conn, &token, None) + .is_err()); + + // bad macaroon (wrong signing identifier) + let c_other_location = AuthConfectionary::new( + "test.fatcat.wiki".to_string(), + "other-dummy-wrong".to_string(), + &other_key, + ) + .expect("creating dummy AuthConfectionary"); + let token_other_location = c_other_location.create_token(editor_id, None).unwrap(); + assert!(c_other_location + .parse_macaroon_token(&conn, &token_other_location, None) + .is_ok()); + assert!(c_other_key + .parse_macaroon_token(&conn, &token_other_location, None) + .is_err()); + assert!(c_other_location + .parse_macaroon_token(&conn, &token_other_key, None) + .is_err()); } |