diff options
| -rw-r--r-- | .gitlab-ci.yml | 16 | ||||
| -rw-r--r-- | python/sandcrawler/workers.py | 2 |
2 files changed, 12 insertions, 6 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7792992..5053a57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,10 @@ test_python_hadoop: - pipenv install --dev --deploy - pipenv run pytest --cov +# needs fixing; some upstream com.hadoop.gplcompression#hadoop-lzo;0.4.16: java.lang.NullPointerException +# change happened test_scalding: + when: manual script: - ./please -h - cd scalding @@ -33,9 +36,10 @@ test_scalding: - sbt -mem 1024 assembly # Needs fixing -#test_pig: -# script: -# - ./fetch_hadoop.sh -# - cd pig -# - pipenv install --dev --deploy -# - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest +test_pig: + when: manual + script: + - ./fetch_hadoop.sh + - cd pig + - pipenv install --dev --deploy + - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index a23d9a4..25d567f 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -108,6 +108,8 @@ class KafkaSink(SandcrawlerWorker): config = self.producer_config({ 'bootstrap.servers': kafka_hosts, 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes + 'api.version.request': True, + 'api.version.fallback.ms': 0, }) self.producer = Producer(config) |