diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-06-25 11:46:06 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-06-25 11:46:06 -0700 |
| commit | 413aa7b3797f52e7f87ea83cd64a5212aef6af9b (patch) | |
| tree | c3c641e228aa6ecd2a0c59d36387cc854eadf96c | |
| parent | 3a6469f89261b9bbcd28632094a349f66ef27ebb (diff) | |
| download | chocula-413aa7b3797f52e7f87ea83cd64a5212aef6af9b.tar.gz chocula-413aa7b3797f52e7f87ea83cd64a5212aef6af9b.zip | |
improvements to Makefile stats/status commands
| -rw-r--r-- | Makefile | 4 |
1 files changed, 2 insertions, 2 deletions
@@ -103,12 +103,12 @@ update-sources: data/$(TODAY)/kbart_JSTOR.txt data/$(TODAY)/kbart_CLOCKSS.txt da data/$(TODAY)/homepage_status.json: pipenv run python -m chocula export_urls | shuf | pv -l > /tmp/chocula_urls.tsv - pipenv run parallel -j10 --bar --pipepart -a /tmp/chocula_urls.tsv ./check_issn_urls.py > /tmp/homepage_status.json + pipenv run parallel -j10 --pipepart --line-buffer -a /tmp/chocula_urls.tsv ./check_issn_urls.py | pv -l > /tmp/homepage_status.json mv /tmp/url_status.json $@ data/$(TODAY)/container_stats.json: data/container_export.json cat data/container_export.json | jq .issnl -r | sort -u > /tmp/container_issnl.tsv - cat /tmp/container_issnl.tsv | parallel -j10 curl -s 'https://fatcat.wiki/container/issnl/{}/stats.json' | jq -c . > /tmp/container_stats.json + cat /tmp/container_issnl.tsv | parallel -j10 curl --fail -s 'https://fatcat.wiki/container/issnl/{}/stats.json' | jq -c . | pv -l > /tmp/container_stats.json cp /tmp/container_stats.json $@ .PHONY: upload-sources |