aboutsummaryrefslogtreecommitdiffstats
path: root/pig/tests
diff options
context:
space:
mode:
Diffstat (limited to 'pig/tests')
-rw-r--r--pig/tests/log4j.properties9
-rw-r--r--pig/tests/pig.properties2
-rw-r--r--pig/tests/pighelper.py21
3 files changed, 24 insertions, 8 deletions
diff --git a/pig/tests/log4j.properties b/pig/tests/log4j.properties
new file mode 100644
index 0000000..e094722
--- /dev/null
+++ b/pig/tests/log4j.properties
@@ -0,0 +1,9 @@
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+# With these enabled, get "log4j:ERROR Attempted to append to closed appender named [stdout]"
+#log4j.logger.org.apache.pig=WARN, stdout
+#log4j.logger.org.apache.hadoop = WARN, stdout
diff --git a/pig/tests/pig.properties b/pig/tests/pig.properties
new file mode 100644
index 0000000..082b936
--- /dev/null
+++ b/pig/tests/pig.properties
@@ -0,0 +1,2 @@
+log4jconf=./tests/log4j.properties
+stop.on.failure=true
diff --git a/pig/tests/pighelper.py b/pig/tests/pighelper.py
index cee074d..3786a07 100644
--- a/pig/tests/pighelper.py
+++ b/pig/tests/pighelper.py
@@ -1,9 +1,17 @@
"""
A helper class for locally testing Pig scripts.
+Include `PigTestHelper` and extend in your test classes, call `self.run_pig()`
+with your script and example input file, then look at the output (at returned
+path) to check for validity.
+
+TODO: squelch pig stdout going to console, presumably because of subprocess.run
+behavior
+
author: Bryan Newbold <bnewbold@archive.org>
"""
import os
+import shutil
import tempfile
import unittest
import subprocess
@@ -16,30 +24,26 @@ class PigTestHelper(unittest.TestCase):
def setUpClass(cls):
cls._pigpath= "./deps/pig/bin/pig"
+ cls._classpath = "./deps/hadoop/share/hadoop/common/lib"
cls._base = [cls._pigpath,
'-x', 'local',
- '-log4jconf', 'pig_log4j.properties',
- '-stop_on_failure']
+ '-P', './tests/pig.properties']
# Check that pig is functioning
if subprocess.call(cls._base + ['-version']) != 0:
raise unittest.SkipTest("Failed to find and run Pig")
- # Classpath?
- # os.path.join("pig-0.12.0-cdh5.0.1", "pig.jar"),
- # os.path.join("pig-0.12.0-cdh5.0.1", "lib", "*"),
- # "hadoop-2.3.0-cdh5.0.1"
-
def setUp(self):
self._tmpdir = tempfile.mkdtemp()
def tearDown(self):
- os.rmdir(self._tmpdir)
+ shutil.rmtree(self._tmpdir)
def run_pig_raw(self, params):
"""Low-level variant with params appended directly. Returns
CompletedProcess, raises an error if return value isn't succes"""
+ print("Running: {}".format(' '.join(self._base + params)))
retval = subprocess.run(self._base + params,
timeout=20.0,
check=True)
@@ -67,3 +71,4 @@ class PigTestHelper(unittest.TestCase):
self.run_pig_raw(params)
return out_file
+ # TODO: helper to verify that output matches an expected file