aboutsummaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/harvest
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-04-08 15:04:50 -0700
committerBryan Newbold <bnewbold@robocracy.org>2019-09-20 11:21:10 -0700
commit2e6cbe2d2ee444c465351b5af4f3f1455bc42bb5 (patch)
tree2b28c17010ab041030319e3b5d29e1e37a7815c8 /python/fatcat_tools/harvest
parent47a90c135b06c18d38a1c92384ff474cb262d5d8 (diff)
downloadfatcat-2e6cbe2d2ee444c465351b5af4f3f1455bc42bb5.tar.gz
fatcat-2e6cbe2d2ee444c465351b5af4f3f1455bc42bb5.zip
bump max message size to ~20 MBytes
Diffstat (limited to 'python/fatcat_tools/harvest')
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py1
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py1
2 files changed, 2 insertions, 0 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index a5d9a04d..abf92792 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -57,6 +57,7 @@ class HarvestCrossrefWorker:
self.kafka_config = {
'bootstrap.servers': kafka_hosts,
'delivery.report.only.error': True,
+ 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes
'default.topic.config': {
'request.required.acks': 'all',
},
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index 129df2e1..a0c3c2cf 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -40,6 +40,7 @@ class HarvestOaiPmhWorker:
self.kafka_config = {
'bootstrap.servers': kafka_hosts,
'delivery.report.only.error': True,
+ 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes
'default.topic.config':
{'request.required.acks': 'all'},
}