QA test import notes

1 files changed, 101 insertions, 0 deletions

diff --git a/notes/bootstrap/import_timing_20190128.txt b/notes/bootstrap/import_timing_20190128.txt
new file mode 100644
index 00000000..b5e49067
--- /dev/null
+++ b/notes/bootstrap/import_timing_20190128.txt
@@ -0,0 +1,101 @@
+
+    # shutdown all the import/export/etc
+
+    sudo service fatcat-web stop
+    sudo service fatcat-api stop
+
+    # ansible playbook push
+
+    # as postgres user:
+    DATABASE_URL=postgres://postgres@/fatcat_prod /opt/cargo/bin/diesel database reset
+    sudo service postgresql restart
+
+    http delete :9200/fatcat_release
+    http delete :9200/fatcat_container
+    http delete :9200/fatcat_changelog
+    http put :9200/fatcat_release < release_schema.json
+    http put :9200/fatcat_container < container_schema.json
+    http put :9200/fatcat_changelog < changelog_schema.json
+    sudo service elasticsearch stop
+    sudo service kibana stop
+
+    sudo service postgresql start
+    sudo service fatcat-api start
+
+    # ensure rust/.env -> /srv/fatcat/config/fatcat_api.env
+    wget https://archive.org/download/ia_journal_metadata/journal_metadata.2019-01-25.json
+
+    # after running below imports
+    sudo service fatcat-web start
+    sudo service elasticsearch start
+    sudo service kibana start
+
+## QA Dry Run
+
+Git commit: 4137d2e8303999dedca404c2a369e7a8b6daf6ca
+      then: e23d62ed0fd65c5cde1d367569bee4f995b38571
+      then: d7d42a21b0b652496d26a10457a23fe6b615da90
+      then: 0a427dae89357bef0c45830b22b5f18e894747ba
+
+    # as webcrawl
+    ../extra/bootstrap_bots.sh
+
+    export FATCAT_AUTH_WORKER_JOURNAL_METADATA="..."
+    time ./fatcat_import.py journal-metadata /srv/fatcat/datasets/journal_metadata.2019-01-25.json
+
+        Counter({'total': 107869, 'insert': 107832, 'skip': 35, 'exists': 2, 'update': 0})
+        real    4m53.194s
+        user    2m0.168s
+        sys     0m4.760s
+
+    export FATCAT_AUTH_WORKER_ORCID="..."
+    time parallel --bar --pipepart -j8 -a /srv/fatcat/datasets/public_profiles_1_2_json.all.json ./fatcat_import.py orcid -
+
+        Counter({'total': 48097, 'insert': 47912, 'skip': 185, 'update': 0, 'exists': 0}) (times ~8x)
+        real    28m9.579s
+        user    88m58.112s
+        sys     5m0.352s
+
+    # XXX: THIS IS ONLY 1 MILLION
+    export FATCAT_AUTH_WORKER_CROSSREF="..."
+    time zcat /srv/fatcat/datasets/crossref-works.2018-09-05.1mil.json.gz | time parallel -j20 --round-robin --pipe ./fatcat_import.py crossref --bezerk-mode - /srv/fatcat/datasets/20181203.ISSN-to-ISSN-L.txt --extid-map-file /srv/fatcat/datasets/release_ids.ia_munge_20180908.sqlite3
+
+
+
+    export FATCAT_AUTH_SANDCRAWLER="..."
+    export FATCAT_API_AUTH_TOKEN=$FATCAT_AUTH_SANDCRAWLER
+    time zcat /srv/fatcat/datasets/ia_papers_manifest_2018-01-25.matched.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py matched --bezerk-mode -
+    # ran for a bit
+
+    time zcat /srv/fatcat/datasets/2018-08-27-2352.17-matchcrossref.insertable.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py matched -
+    # ran for a bit
+
+    time zcat /srv/fatcat/datasets/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py grobid-metadata --bezerk-mode -
+    # ran for a bit
+
+    # XXX: FULL SIZE
+    export FATCAT_AUTH_WORKER_CROSSREF="..."
+	time xzcat /srv/fatcat/datasets/crossref-works.2018-09-05.json.xz | time parallel -j20 --round-robin --pipe ./fatcat_import.py crossref - /srv/fatcat/datasets/20181203.ISSN-to-ISSN-L.txt --extid-map-file /srv/fatcat/datasets/release_ids.ia_munge_20180908.sqlite3
+
+TODO:
+x bezerk mode is broken (at least for crossref, on command line; kwargs?)
+x either delete example entities, or delete/update them after import
+x unexpected database error: new row for relation \"abstracts\" violates check constraint \"abstracts_content_check\"
+x {"success":false,"error":"NotFound","message":"no such unknown found: N/A"}
+    => this seems to mean auth failed (but bad message)
+x ansible: /srv/fatcat/snapshots should be world-writable
+x HTTP response body: {"success":false,"error":"BadRequest","message":"broke a constraint or made an otherwise invalid request: display_name is required for all Creator entities"}
+x TypeError: __init__() missing 1 required positional argument: 'json_file'
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_rev\" violates check constraint \"release_rev_title_check\""}
+x backwards rel/url for some (but not all) file URLs:
+    {"url":"web","rel":"http://www.spe.ut.ee/ojs-2.2.2/index.php/spe/article/download/spe.2012.5.2.07/74/"}
+x 2019/01/29 04:55:14 failed to json decode doc: invalid character '\'' looking for beginning of object key string
+    => this was due to random output coming out of python into esbulk (and getting mangled)
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_rev\" violates check constraint \"release_rev_original_title_check\""}
+    => fixed in 'clean()' for most/all such fields
+x HTTP response body: {"success":false,"error":"ConstraintViolation","message":"unexpected database error: new row for relation \"release_contrib\" violates check constraint \"release_contrib_raw_affiliation_check\""}
+x   File "/srv/fatcat/src/python/fatcat_tools/importers/crossref.py", line 236, in parse_record
+    extra['journal-title'] = rm['journal-title']
+    UnboundLocalError: local variable 'extra' referenced before assignment
+x release_rev_publisher_check
+x release_contrib_raw_affiliation_check