From e68d43e2369eed7ddf288be8c8f2edd0a85974e1 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Thu, 5 Apr 2018 19:24:07 -0700
Subject: make happybase mock injection slightly less horrible

---
 mapreduce/backfill_hbase_from_cdx.py            | 28 +++++++++++--------------
 mapreduce/extraction_cdx_grobid.py              | 28 +++++++++++--------------
 mapreduce/tests/test_backfill_hbase_from_cdx.py |  5 +++--
 mapreduce/tests/test_extraction_cdx_grobid.py   |  6 ++++--
 4 files changed, 31 insertions(+), 36 deletions(-)

(limited to 'mapreduce')

diff --git a/mapreduce/backfill_hbase_from_cdx.py b/mapreduce/backfill_hbase_from_cdx.py
index 57c18e4..2643195 100755
--- a/mapreduce/backfill_hbase_from_cdx.py
+++ b/mapreduce/backfill_hbase_from_cdx.py
@@ -43,27 +43,23 @@ class MRCDXBackfillHBase(MRJob):
                               help='HBase thrift API host to connect to')
 
     def __init__(self, *args, **kwargs):
-
-        # Allow passthrough for tests
-        if 'hb_table' in kwargs:
-            self.hb_table = kwargs.pop('hb_table')
-        else:
-            self.hb_table = None
-
         super(MRCDXBackfillHBase, self).__init__(*args, **kwargs)
         self.mime_filter = ['application/pdf']
+        self.hb_table = None
 
     def mapper_init(self):
 
-        if self.hb_table is None:
-            try:
-                host = self.options.hbase_host
-                # TODO: make these configs accessible from... mrconf.cfg?
-                hb_conn = happybase.Connection(host=host, transport="framed",
-                    protocol="compact")
-            except Exception as err:
-                raise Exception("Couldn't connect to HBase using host: {}".format(host))
-            self.hb_table = hb_conn.table(self.options.hbase_table)
+        if self.hb_table:
+            return
+
+        try:
+            host = self.options.hbase_host
+            # TODO: make these configs accessible from... mrconf.cfg?
+            hb_conn = happybase.Connection(host=host, transport="framed",
+                protocol="compact")
+        except Exception as err:
+            raise Exception("Couldn't connect to HBase using host: {}".format(host))
+        self.hb_table = hb_conn.table(self.options.hbase_table)
 
     def mapper(self, _, raw_cdx):
 
diff --git a/mapreduce/extraction_cdx_grobid.py b/mapreduce/extraction_cdx_grobid.py
index 0ba95e6..a4a13f8 100755
--- a/mapreduce/extraction_cdx_grobid.py
+++ b/mapreduce/extraction_cdx_grobid.py
@@ -53,15 +53,9 @@ class MRExtractCdxGrobid(MRJob):
                               help='URI of GROBID API Server')
 
     def __init__(self, *args, **kwargs):
-
-        # Allow passthrough for tests
-        if 'hb_table' in kwargs:
-            self.hb_table = kwargs.pop('hb_table')
-        else:
-            self.hb_table = None
-
         super(MRExtractCdxGrobid, self).__init__(*args, **kwargs)
         self.mime_filter = ['application/pdf']
+        self.hb_table = None
 
     def grobid_process_fulltext(self, content):
         r = requests.post(self.options.grobid_uri + "/api/processFulltextDocument",
@@ -73,15 +67,17 @@ class MRExtractCdxGrobid(MRJob):
 
     def mapper_init(self):
 
-        if self.hb_table is None:
-            try:
-                host = self.options.hbase_host
-                # TODO: make these configs accessible from... mrconf.cfg?
-                hb_conn = happybase.Connection(host=host, transport="framed",
-                    protocol="compact")
-            except Exception as err:
-                raise Exception("Couldn't connect to HBase using host: {}".format(host))
-            self.hb_table = hb_conn.table(self.options.hbase_table)
+        if self.hb_table:
+            return
+
+        try:
+            host = self.options.hbase_host
+            # TODO: make these configs accessible from... mrconf.cfg?
+            hb_conn = happybase.Connection(host=host, transport="framed",
+                protocol="compact")
+        except Exception as err:
+            raise Exception("Couldn't connect to HBase using host: {}".format(host))
+        self.hb_table = hb_conn.table(self.options.hbase_table)
 
     def parse_line(self, raw_cdx):
 
diff --git a/mapreduce/tests/test_backfill_hbase_from_cdx.py b/mapreduce/tests/test_backfill_hbase_from_cdx.py
index 2dbbc25..070662b 100644
--- a/mapreduce/tests/test_backfill_hbase_from_cdx.py
+++ b/mapreduce/tests/test_backfill_hbase_from_cdx.py
@@ -15,12 +15,13 @@ def job():
     Note: this mock only seems to work with job.run_mapper(), not job.run();
     the later results in a separate instantiation without the mock?
     """
+    job = MRCDXBackfillHBase(['--no-conf', '-'])
+
     conn = happybase_mock.Connection()
     conn.create_table('wbgrp-journal-extract-test',
         {'file': {}, 'grobid0': {}, 'f': {}})
-    table = conn.table('wbgrp-journal-extract-test')
+    job.hb_table = conn.table('wbgrp-journal-extract-test')
 
-    job = MRCDXBackfillHBase(['--no-conf', '-'], hb_table=table)
     return job
 
 
diff --git a/mapreduce/tests/test_extraction_cdx_grobid.py b/mapreduce/tests/test_extraction_cdx_grobid.py
index 46a89aa..02d2b41 100644
--- a/mapreduce/tests/test_extraction_cdx_grobid.py
+++ b/mapreduce/tests/test_extraction_cdx_grobid.py
@@ -18,14 +18,16 @@ def job():
     Note: this mock only seems to work with job.run_mapper(), not job.run();
     the later results in a separate instantiation without the mock?
     """
+    job = MRExtractCdxGrobid(['--no-conf', '-'])
+
     conn = happybase_mock.Connection()
     conn.create_table('wbgrp-journal-extract-test',
         {'file': {}, 'grobid0': {}, 'f': {}})
-    table = conn.table('wbgrp-journal-extract-test')
+    job.hb_table = conn.table('wbgrp-journal-extract-test')
 
-    job = MRExtractCdxGrobid(['--no-conf', '-'], hb_table=table)
     return job
 
+
 @mock.patch('extraction_cdx_grobid.MRExtractCdxGrobid.fetch_warc_content', return_value=(FAKE_PDF_BYTES, None))
 @responses.activate
 def test_mapper_lines(mock_fetch, job):
-- 
cgit v1.2.3