summaryrefslogtreecommitdiffstats
path: root/notes/bootstrap
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-11-02 11:39:25 -0700
committerBryan Newbold <bnewbold@robocracy.org>2018-11-02 11:39:25 -0700
commit70d1f092df8e7d73d556587cac5bfe4efd2196bd (patch)
tree0e8f77827eefb5ff8438f792e374fb6be8c51255 /notes/bootstrap
parent233fe19ec028b2703f75cca3fb02939ed7c79d25 (diff)
downloadfatcat-70d1f092df8e7d73d556587cac5bfe4efd2196bd.tar.gz
fatcat-70d1f092df8e7d73d556587cac5bfe4efd2196bd.zip
backup notes
Diffstat (limited to 'notes/bootstrap')
-rw-r--r--notes/bootstrap/import_timing_20180924.txt135
1 files changed, 135 insertions, 0 deletions
diff --git a/notes/bootstrap/import_timing_20180924.txt b/notes/bootstrap/import_timing_20180924.txt
new file mode 100644
index 00000000..d9c49fc2
--- /dev/null
+++ b/notes/bootstrap/import_timing_20180924.txt
@@ -0,0 +1,135 @@
+
+Pretty much all imports done at git hash c1d0fea
+
+ time xzcat /srv/fatcat/datasets/crossref-works.2018-09-05.json.xz | time parallel -j20 --round-robin --pipe ./fatcat_import.py import-crossref - /srv/fatcat/datasets/20180216.ISSN-to-ISSN-L.txt /srv/fatcat/datasets/release_ids.ia_munge_20180908.sqlite3
+
+ Processed 4990450 lines, inserted 4005034, updated 0. (etc)
+
+ 133387.36user 5255.64system 24:19:01elapsed 158%CPU (0avgtext+0avgdata 448196maxresident)k
+ 177480808inputs+432403768outputs (204major+48533880minor)pagefaults 0swaps
+
+ real 1459m1.518s
+ user 2308m24.300s
+ sys 93m17.132s
+
+ Longer, bigger, etc than previously!
+
+ Size: 377.49G
+
+ select count(id) from release_ident; => 79,880,900
+
+
+ zcat /srv/fatcat/datasets/ia_papers_manifest_2018-01-25.matched.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py import-matched --no-file-update -
+ Processed 531700 lines, inserted 511751, updated 0.
+ Command exited with non-zero status 1
+ 17087.60user 717.77system 3:07:11elapsed 158%CPU (0avgtext+0avgdata 67420maxresident)k
+ 60128inputs+3401960outputs (141major+403282minor)pagefaults 0swaps
+
+
+Sample of "not found" DOIs:
+
+ DOI not found: 10.1109/mic.2005.100
+ DOI not found: 10.3386/w9732
+ DOI not found: 10.1090/s0002-9939-97-04114-2
+ DOI not found: 10.1186/1475-2867-5-29
+ DOI not found: 10.2172/143964
+ DOI not found: 10.2172/10170724
+ DOI not found: 10.2172/383051
+ DOI not found: 10.1017/s0033291700051370
+ DOI not found: 10.12980/jclm.3.2015j5-154
+ DOI not found: 10.2172/801341
+ DOI not found: 10.2172/899508
+
+ DOI not found: 10.1136/bmj.2.4570.302
+ DOI not found: 10.1136/bmj.2.4687.1049
+ DOI not found: 10.1163/221125903x00429
+ DOI not found: 10.1177/004947557800800102
+ DOI not found: 10.1177/107755874800500313
+ DOI not found: 10.1177/107755874800500415
+ DOI not found: 10.1177/107755874800500713
+
+ DOI not found: 10.5990/jwpa.29.72
+ DOI not found: 10.2307/1107183
+ DOI not found: 10.1101/147165
+ DOI not found: 10.17848/wp04-108
+ DOI not found: 10.2172/542039
+ DOI not found: 10.2172/542040
+ DOI not found: 10.1002/9781444308747.ch6
+
+
+ zcat /srv/fatcat/datasets/2018-08-27-2352.17-matchcrossref.insertable.json.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py import-matched -
+ Processed 485400 lines, inserted 283498, updated 197825.
+ 25649.33user 1152.84system 4:42:24elapsed 158%CPU (0avgtext+0avgdata 38984maxresident)k
+ 38584inputs+2371576outputs (136major+357478minor)pagefaults 0swaps
+
+ Size: 395.13G
+
+ table_name | table_size | indexes_size | total_size
+--------------------------------------------------------------+------------+--------------+------------
+ "public"."release_ref" | 154 GB | 54 GB | 208 GB
+ "public"."release_rev" | 39 GB | 22 GB | 61 GB
+ "public"."release_contrib" | 25 GB | 22 GB | 47 GB
+ "public"."release_edit" | 7095 MB | 6956 MB | 14 GB
+ "public"."work_edit" | 7095 MB | 6956 MB | 14 GB
+ "public"."release_ident" | 5203 MB | 6254 MB | 11 GB
+ "public"."work_ident" | 5203 MB | 6254 MB | 11 GB
+ "public"."file_rev_url" | 6535 MB | 2478 MB | 9013 MB
+ "public"."work_rev" | 3376 MB | 3127 MB | 6503 MB
+ "public"."file_rev" | 1404 MB | 2115 MB | 3519 MB
+ "public"."abstracts" | 2611 MB | 208 MB | 2820 MB
+ "public"."file_edit" | 1089 MB | 1066 MB | 2155 MB
+ "public"."file_release" | 713 MB | 1250 MB | 1962 MB
+ "public"."file_ident" | 618 MB | 740 MB | 1358 MB
+ "public"."creator_rev" | 371 MB | 457 MB | 828 MB
+ "public"."creator_edit" | 347 MB | 352 MB | 699 MB
+ "public"."release_rev_abstract" | 284 MB | 369 MB | 653 MB
+ "public"."creator_ident" | 255 MB | 305 MB | 560 MB
+ "public"."changelog" | 138 MB | 142 MB | 279 MB
+ "public"."editgroup" | 155 MB | 92 MB | 247 MB
+ "public"."container_rev" | 20 MB | 9272 kB | 29 MB
+ "public"."container_edit" | 8312 kB | 7360 kB | 15 MB
+ "public"."container_ident" | 7272 kB | 6832 kB | 14 MB
+
+Exports!
+
+ time cat /tmp/fatcat_ident_releases.tsv | ./target/release/fatcat-export release --expand files,container -j8 | gzip > release_export_expanded.json.gz
+ INFO 2018-09-27T22:54:30Z: fatcat_export: Done reading (79880900 lines), waiting for workers to exit...
+
+ real 384m29.435s
+ user 740m1.060s
+ sys 229m11.632s
+
+
+ time zcat /srv/fatcat/snapshots/2018-09-24/release_export_expanded.json.gz | ./transform_release.py | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat -type release
+ 2018/09/28 02:56:36 79880900 docs in 2h56m48.425914042s at 7529.948 docs/s with 8 workers
+ 2018/09/28 02:56:36 applied setting: {"index": {"refresh_interval": "1s"}} with status 200 OK
+ 2018/09/28 02:56:36 applied setting: {"index": {"number_of_replicas": "1"}} with status 200 OK
+ 2018/09/28 02:56:40 index flushed: 200 OK
+
+ real 176m53.138s
+ user 318m17.004s
+ sys 29m48.944s
+
+ webcrawl@wbgrp-svc503:/srv/fatcat/src/extra/elasticsearch$ du -sh /srv/elasticsearch/data/
+ 52G /srv/elasticsearch/data/
+
+TODO:
+x abstracts
+x file_hashes
+x ext idents
+x upload to an item
+x download and re-build elastic
+- insert new mellon matches
+
+
+ time zcat /srv/fatcat/datasets/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz | pv -l | time parallel -j12 --round-robin --pipe ./fatcat_import.py import-grobid-metadata -
+ [...]
+ Processed 132994 lines, inserted 123052, updated 0.
+ Processed 132984 lines, inserted 122979, updated 0.
+ 10930.34user 475.87system 2:40:03elapsed 118%CPU (0avgtext+0avgdata 68180maxresident)k
+ 8912inputs+20157832outputs (59major+1104467minor)pagefaults 0swaps
+
+ real 160m3.573s
+ user 184m54.176s
+ sys 8m23.388s
+