From 21e9fd509b63182657d1b76b02c2921b2a5334b0 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 15 Nov 2019 15:52:53 -0800
Subject: CI: make some jobs manual

Scalding test is broken :(
But we aren't even using that code much these days.
---
 .gitlab-ci.yml                | 16 ++++++++++------
 python/sandcrawler/workers.py |  2 ++
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7792992..5053a57 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,10 @@ test_python_hadoop:
     - pipenv install --dev --deploy
     - pipenv run pytest --cov
 
+# needs fixing; some upstream com.hadoop.gplcompression#hadoop-lzo;0.4.16: java.lang.NullPointerException
+# change happened
 test_scalding:
+  when: manual
   script:
     - ./please -h
     - cd scalding
@@ -33,9 +36,10 @@ test_scalding:
     - sbt -mem 1024 assembly
 
 # Needs fixing
-#test_pig:
-#  script:
-#    - ./fetch_hadoop.sh
-#    - cd pig
-#    - pipenv install --dev --deploy
-#    - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest
+test_pig:
+  when: manual
+  script:
+    - ./fetch_hadoop.sh
+    - cd pig
+    - pipenv install --dev --deploy
+    - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index a23d9a4..25d567f 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -108,6 +108,8 @@ class KafkaSink(SandcrawlerWorker):
         config = self.producer_config({
             'bootstrap.servers': kafka_hosts,
             'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes
+            'api.version.request': True,
+            'api.version.fallback.ms': 0,
         })
         self.producer = Producer(config)
 
-- 
cgit v1.2.3