diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-11-15 15:52:53 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-11-15 15:52:56 -0800 |
| commit | 21e9fd509b63182657d1b76b02c2921b2a5334b0 (patch) | |
| tree | a06953697d8a3785bc065d54f83e053ddd8efb25 | |
| parent | 0e755b499b1d0e8f3a8abce9032896936db3b188 (diff) | |
| download | sandcrawler-21e9fd509b63182657d1b76b02c2921b2a5334b0.tar.gz sandcrawler-21e9fd509b63182657d1b76b02c2921b2a5334b0.zip | |
CI: make some jobs manual
Scalding test is broken :(
But we aren't even using that code much these days.
| -rw-r--r-- | .gitlab-ci.yml | 16 | ||||
| -rw-r--r-- | python/sandcrawler/workers.py | 2 |
2 files changed, 12 insertions, 6 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7792992..5053a57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,10 @@ test_python_hadoop: - pipenv install --dev --deploy - pipenv run pytest --cov +# needs fixing; some upstream com.hadoop.gplcompression#hadoop-lzo;0.4.16: java.lang.NullPointerException +# change happened test_scalding: + when: manual script: - ./please -h - cd scalding @@ -33,9 +36,10 @@ test_scalding: - sbt -mem 1024 assembly # Needs fixing -#test_pig: -# script: -# - ./fetch_hadoop.sh -# - cd pig -# - pipenv install --dev --deploy -# - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest +test_pig: + when: manual + script: + - ./fetch_hadoop.sh + - cd pig + - pipenv install --dev --deploy + - JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") pipenv run pytest diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index a23d9a4..25d567f 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -108,6 +108,8 @@ class KafkaSink(SandcrawlerWorker): config = self.producer_config({ 'bootstrap.servers': kafka_hosts, 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes + 'api.version.request': True, + 'api.version.fallback.ms': 0, }) self.producer = Producer(config) |