aboutsummaryrefslogtreecommitdiffstats
path: root/backfill
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-04-03 02:25:28 +0000
committerBryan Newbold <bnewbold@archive.org>2018-04-03 02:25:28 +0000
commitca2d1bd7eccd619173e1d326c9a2ebb1a2d3d502 (patch)
tree7559cc3080d89c278a32146fdc33c762fd790c3b /backfill
parent630e956c05604aaf8bf5b7154a01ad956b13e440 (diff)
downloadsandcrawler-ca2d1bd7eccd619173e1d326c9a2ebb1a2d3d502.tar.gz
sandcrawler-ca2d1bd7eccd619173e1d326c9a2ebb1a2d3d502.zip
actually running hadoop job on cluster
Diffstat (limited to 'backfill')
-rw-r--r--backfill/README.md14
-rw-r--r--backfill/mrjob.conf4
2 files changed, 18 insertions, 0 deletions
diff --git a/backfill/README.md b/backfill/README.md
index 6af8f33..2e3dc7a 100644
--- a/backfill/README.md
+++ b/backfill/README.md
@@ -10,3 +10,17 @@ running on a devbox:
--hbase-table wbgrp-journal-extract-0-qa \
--hbase-host bnewbold-dev.us.archive.org
+Actual invocation to run on Hadoop cluster (running on an IA devbox, where
+hadoop environment is configured):
+
+ # Create tarball of virtualenv
+ pipenv shell
+ tar -czf backfill-4OmRI0zZ.tar.gz -C /home/bnewbold/.local/share/virtualenvs/backfill-4OmRI0zZ .
+
+ ./backfill_hbase_from_cdx.py \
+ -r hadoop \
+ --hbase-host bnewbold-dev.us.archive.org \
+ --hbase-table wbgrp-journal-extract-0-qa \
+ -c mrjob.conf \
+ --archive backfill-4OmRI0zZ.tar.gz#venv \
+ hdfs:///user/bnewbold/journal_crawl_cdx/citeseerx_crawl_2017.cdx
diff --git a/backfill/mrjob.conf b/backfill/mrjob.conf
new file mode 100644
index 0000000..cb286f1
--- /dev/null
+++ b/backfill/mrjob.conf
@@ -0,0 +1,4 @@
+runners:
+ hadoop:
+ setup:
+ - export PYTHONPATH=$PYTHONPATH:venv/lib/python3.5/site-packages/