diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2019-04-08 15:04:50 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-04-08 15:04:50 -0700 |
| commit | d43ee5f73e1f7d7f6317d1e7dfb1ddd0e510a353 (patch) | |
| tree | 60d5c716c82fe98a8df537ba0c13d2b96a14f1ce /python/fatcat_tools | |
| parent | 953886e6e42294ca81583e9b2c4e2fa16343a86e (diff) | |
| download | fatcat-d43ee5f73e1f7d7f6317d1e7dfb1ddd0e510a353.tar.gz fatcat-d43ee5f73e1f7d7f6317d1e7dfb1ddd0e510a353.zip | |
bump max message size to ~20 MBytes
Diffstat (limited to 'python/fatcat_tools')
| -rw-r--r-- | python/fatcat_tools/harvest/doi_registrars.py | 1 | ||||
| -rw-r--r-- | python/fatcat_tools/harvest/oaipmh.py | 1 |
2 files changed, 2 insertions, 0 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index a5d9a04d..abf92792 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -57,6 +57,7 @@ class HarvestCrossrefWorker: self.kafka_config = { 'bootstrap.servers': kafka_hosts, 'delivery.report.only.error': True, + 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes 'default.topic.config': { 'request.required.acks': 'all', }, diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index 129df2e1..a0c3c2cf 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -40,6 +40,7 @@ class HarvestOaiPmhWorker: self.kafka_config = { 'bootstrap.servers': kafka_hosts, 'delivery.report.only.error': True, + 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes 'default.topic.config': {'request.required.acks': 'all'}, } |