deliver_dumpgrobid_to_s3: storage class config

1 files changed, 7 insertions, 1 deletions

diff --git a/python/deliver_dumpgrobid_to_s3.py b/python/deliver_dumpgrobid_to_s3.py
index 7fb0f08..6a9aea6 100755
--- a/python/deliver_dumpgrobid_to_s3.py
+++ b/python/deliver_dumpgrobid_to_s3.py
@@ -46,6 +46,7 @@ class DeliverDumpGrobidS3():
         self.s3_bucket = s3_bucket
         self.s3_prefix = kwargs.get('s3_prefix', 'grobid/')
         self.s3_suffix = kwargs.get('s3_suffix', '.tei.xml')
+        self.s3_storage_class = kwargs.get('s3_storage_class', 'STANDARD')
         self.s3 = boto3.resource('s3')
         self.bucket = self.s3.Bucket(self.s3_bucket)
 
@@ -71,7 +72,8 @@ class DeliverDumpGrobidS3():
                     self.s3_prefix,
                     sha1_hex[0:4],
                     sha1_hex,
-                    self.s3_suffix),
+                    self.s3_suffix,
+                    StorageClass=self.s3_storage_class),
                 Body=tei_xml)
             print("{}\tsuccess\t{}\t{}".format(sha1_hex, obj.key, len(tei_xml)))
             self.count['success-s3'] += 1
@@ -93,6 +95,10 @@ def main():
                         type=str,
                         default=".tei.xml",
                         help='file suffix for created objects')
+    parser.add_argument('--s3-storage-class',
+                        type=str,
+                        default="STANDARD",
+                        help='AWS S3 storage class (redundancy) to use')
     parser.add_argument('dump_file',
                         help="TSV/JSON dump file",
                         default=sys.stdin,