aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--notes/bootstrap/import_timing_20190128.txt101
1 files changed, 101 insertions, 0 deletions
diff --git a/notes/bootstrap/import_timing_20190128.txt b/notes/bootstrap/import_timing_20190128.txt
new file mode 100644
index 00000000..b5e49067
--- /dev/null
+++ b/notes/bootstrap/import_timing_20190128.txt
@@ -0,0 +1,101 @@
+
+ # shutdown all the import/export/etc
+
+ sudo service fatcat-web stop
+ sudo service fatcat-api stop
+
+ # ansible playbook push
+
+ # as postgres user:
+ DATABASE_URL=postgres://postgres@/fatcat_prod /opt/cargo/bin/diesel database reset
+ sudo service postgresql restart
+
+ http delete :9200/fatcat_release
+ http delete :9200/fatcat_container
+ http delete :9200/fatcat_changelog
+ http put :9200/fatcat_release < release_schema.json
+ http put :9200/fatcat_container < container_schema.json
+ http put :9200/fatcat_changelog < changelog_schema.json
+ sudo service elasticsearch stop
+ sudo service kibana stop
+
+ sudo service postgresql start
+ sudo service fatcat-api start
+
+ # ensure rust/.env -> /srv/fatcat/config/fatcat_api.env
+ wget https://archive.org/download/ia_journal_metadata/journal_metadata.2019-01-25.json
+
+ # after running below imports
+ sudo service fatcat-web start
+ sudo service elasticsearch start
+ sudo service kibana start
+
+## QA Dry Run
+
+Git commit: 4137d2e8303999dedca404c2a369e7a8b6daf6ca
+ then: e23d62ed0fd65c5cde1d367569bee4f995b38571
+ then: d7d42a21b0b652496d26a10457a23fe6b615da90
+ then: 0a427dae89357bef0c45830b22b5f18e894747ba
+
+ # as webcrawl
+ ../extra/bootstrap_bots.sh
+
+ export FATCAT_AUTH_WORKER_JOURNAL_METADATA="..."
+ time ./fatcat_import.py journal-metadata /srv/fatcat/datasets/journal_metadata.2019-01-25.json
+
+ Counter({'total': 107869, 'insert': 107832, 'skip': 35, 'exists': 2, 'update': 0})
+ real 4m53.194s
+ user 2m0.168s
+ sys 0m4.760s
+
+ export FATCAT_AUTH_WORKER_ORCID="..."
+ time parallel --bar --pipepart -j8 -a /srv/fatcat/datasets/public_profiles_1_2_json.all.json ./fatcat_import.py orcid -
+
+ Counter({'total': 48097, 'insert': 47912, 'skip': 185, 'update': 0, 'exists': 0}) (times ~8x)
+ real 28m9.579s
+ user 88m58.112s
+ sys 5m0.352s
+
+ # XXX: THIS IS ONLY 1 MILLION
+ export FATCAT_AUTH_WORKER_CROSSREF="..."
+ time zcat /srv/fatcat/datasets/crossref-works.2018-09-05.1mil.json.gz | time parallel -j20 --round-robin --pipe ./fatcat_import.py crossref --bezerk-mode - /srv/fatcat/datasets/20181203.ISSN-to-ISSN-L.txt --extid-map-file /srv/fatcat/datasets/release_ids.ia_munge_20180908.sqlite3
+
+
+
+ export FATCAT_AUTH_SANDCRAWLER="..."
+ export FATCAT_API_AUTH_TOKEN=$FATCAT_AUTH_SANDCRAWLER
+ time zcat /srv/fatcat/datasets/ia_papers_manifest_2018-01-25.matched.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py matched --bezerk-mode -
+ # ran for a bit
+
+ time zcat /srv/fatcat/datasets/2018-08-27-2352.17-matchcrossref.insertable.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py matched -
+ # ran for a bit
+
+ time zcat /srv/fatcat/datasets/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py grobid-metadata --bezerk-mode -
+ # ran for a bit
+
+ # XXX: FULL SIZE
+ export FATCAT_AUTH_WORKER_CROSSREF="..."
+ time xzcat /srv/fatcat/datasets/crossref-works.2018-09-05.json.xz | time parallel -j20 --round-robin --pipe ./fatcat_import.py crossref - /srv/fatcat/datasets/20181203.ISSN-to-ISSN-L.txt --extid-map-file /srv/fatcat/datasets/release_ids.ia_munge_20180908.sqlite3
+
+TODO:
+x bezerk mode is broken (at least for crossref, on command line; kwargs?)
+x either delete example entities, or delete/update them after import
+x unexpected database error: new row for relation \"abstracts\" violates check constraint \"abstracts_content_check\"
+x {"success":false,"error":"NotFound","message":"no such unknown found: N/A"}
+ => this seems to mean auth failed (but bad message)
+x ansible: /srv/fatcat/snapshots should be world-writable
+x HTTP response body: {"success":false,"error":"BadRequest","message":"broke a constraint or made an otherwise invalid request: display_name is required for all Creator entities"}
+x TypeError: __init__() missing 1 required positional argument: 'json_file'
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_rev\" violates check constraint \"release_rev_title_check\""}
+x backwards rel/url for some (but not all) file URLs:
+ {"url":"web","rel":"http://www.spe.ut.ee/ojs-2.2.2/index.php/spe/article/download/spe.2012.5.2.07/74/"}
+x 2019/01/29 04:55:14 failed to json decode doc: invalid character '\'' looking for beginning of object key string
+ => this was due to random output coming out of python into esbulk (and getting mangled)
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_rev\" violates check constraint \"release_rev_original_title_check\""}
+ => fixed in 'clean()' for most/all such fields
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_contrib\" violates check constraint \"release_contrib_raw_affiliation_check\""}
+x File "/srv/fatcat/src/python/fatcat_tools/importers/crossref.py", line 236, in parse_record
+ extra['journal-title'] = rm['journal-title']
+ UnboundLocalError: local variable 'extra' referenced before assignment
+x release_rev_publisher_check
+x release_contrib_raw_affiliation_check