summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-04-26 23:25:35 -0700
committerBryan Newbold <bnewbold@robocracy.org>2018-04-26 23:25:35 -0700
commit6bae5bd14dfafcee00382d97c8cbc56e75900bac (patch)
tree0681dcaea5f08ff20838f153afaf03765b826f14
parent74e29b596294049a375f7ed8c0f54a987856f8a7 (diff)
downloadfatcat-6bae5bd14dfafcee00382d97c8cbc56e75900bac.tar.gz
fatcat-6bae5bd14dfafcee00382d97c8cbc56e75900bac.zip
api_client: configurable batch size
-rw-r--r--fatcat/api_client.py9
1 files changed, 6 insertions, 3 deletions
diff --git a/fatcat/api_client.py b/fatcat/api_client.py
index f997f5fc..291b7a1b 100644
--- a/fatcat/api_client.py
+++ b/fatcat/api_client.py
@@ -47,12 +47,12 @@ class FatCatApiClient:
self._issn_map[issn] = container_id
return container_id
- def import_crossref_file(self, json_file, create_containers=False):
+ def import_crossref_file(self, json_file, create_containers=False, batchsize=100):
eg = self.new_editgroup()
i = 0
with open(json_file, 'r') as file:
for line in file:
- if i % 1000 == 0:
+ if i % batchsize == 0:
sys.stdout.write('\n{}: '.format(i))
if (i+1) % 20 == 0:
sys.stdout.write('.')
@@ -65,7 +65,10 @@ class FatCatApiClient:
create_containers=create_containers)
except Exception as e:
print("ERROR: {}".format(e))
- if i % 1000 != 0:
+ if i % batchsize == 0:
+ self.accept_editgroup(eg)
+ eg = self.new_editgroup()
+ if i % batchsize != 0:
self.accept_editgroup(eg)
print("done!")