aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitlab-ci.yml16
-rw-r--r--python/sandcrawler/workers.py2
2 files changed, 12 insertions, 6 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7792992..5053a57 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,10 @@ test_python_hadoop:
- pipenv install --dev --deploy
- pipenv run pytest --cov
+# needs fixing; some upstream com.hadoop.gplcompression#hadoop-lzo;0.4.16: java.lang.NullPointerException
+# change happened
test_scalding:
+ when: manual
script:
- ./please -h
- cd scalding
@@ -33,9 +36,10 @@ test_scalding:
- sbt -mem 1024 assembly
# Needs fixing
-#test_pig:
-# script:
-# - ./fetch_hadoop.sh
-# - cd pig
-# - pipenv install --dev --deploy
-# - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest
+test_pig:
+ when: manual
+ script:
+ - ./fetch_hadoop.sh
+ - cd pig
+ - pipenv install --dev --deploy
+ - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index a23d9a4..25d567f 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -108,6 +108,8 @@ class KafkaSink(SandcrawlerWorker):
config = self.producer_config({
'bootstrap.servers': kafka_hosts,
'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes
+ 'api.version.request': True,
+ 'api.version.fallback.ms': 0,
})
self.producer = Producer(config)