2 files changed, 112 insertions, 27 deletions
diff --git a/python/Pipfile b/python/Pipfile
index 16807fe..92c0437 100644
--- a/python/Pipfile
+++ b/python/Pipfile
@@ -21,9 +21,15 @@ mypy = "*"
 flake8 = "*"
 flake8-annotations = "*"
 isort = "*"
-yapf = "*"
 types-requests = "*"
+types-beautifulsoup4 = "*"
 types-dateparser = "*"
+types-psycopg2 = "*"
+types-Pillow = "*"
+
+# must lock black to an exact version because it is still "beta"
+# see: https://github.com/psf/black/issues/517
+black = "==21.9b0"
 
 [packages]
 requests = ">=2"
@@ -39,7 +45,7 @@ ftfy = "*"
 internetarchive = "*"
 Flask = ">=1"
 urlcanon = "*"
-pillow = ">=3"
+Pillow = ">=3"
 python-poppler = ">=0.2.1"
 selectolax = ">=0.2"
 trafilatura = "*"
@@ -49,7 +55,7 @@ braveblock = "*"
 dynaconf = ">=3"
 sentry-sdk = { version = ">=0.14.0", extras = [] }
 zstandard = "*"
-grobid_tei_xml = "0.1.0"
+grobid_tei_xml = "==0.1.*"
 
 [requires]
 python_version = "3.8"
diff --git a/python/Pipfile.lock b/python/Pipfile.lock
index d619752..a1e707d 100644
--- a/python/Pipfile.lock
+++ b/python/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ca9b01c621a8ab8779e1f3d8297e6151c6bf7900c1de28abcfd25fe8d9565389"
+            "sha256": "57f9cc0be1c951ec0c1560b264dbe549463e2ba636e0dc4db32865f0845b24b7"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -58,18 +58,18 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:492962f8cd5ff4aaaaa6bfb321eb526e4758755856226e233b5ead3b3a9c35a5",
-                "sha256:5493c0af5ab79e15f3a37d8b9c86461ac04bae5cb5b82fb911c78dbcfbdab07e"
+                "sha256:eb374e3e5431af55c816757c671f711237325e31177d5ac9441c83a11acbeda9",
+                "sha256:f70abe3ae4e6d4bce6c92f1871347cb4c23a6f6b9dc52261f6342d063cb07ffb"
             ],
             "index": "ia",
-            "version": "==1.19.4"
+            "version": "==1.19.5"
         },
         "botocore": {
             "hashes": [
-                "sha256:579e682b1453fa142dd2f76d09e9c567017f2ae85000329084db8bb49316c228",
-                "sha256:ecc6f90dc27c4d74246a83474edf5ad272e614bebe8c5fb8b9516db4437dbf59"
+                "sha256:69780a647b7fef650fef18cba128795dade71b8c4c8fab57960592f217f9681f",
+                "sha256:ad6a13d820c6b66859b12f0a43a4c5d9ae4fc62c76f3dc81115e0d1f77c2c3fb"
             ],
-            "version": "==1.22.4"
+            "version": "==1.22.5"
         },
         "braveblock": {
             "hashes": [
@@ -367,11 +367,11 @@
         },
         "grobid-tei-xml": {
             "hashes": [
-                "sha256:b0a95b9267d67b5bc431b9243ea64e075ba3cad1d2e051d7b699ad0080936b51",
-                "sha256:b1856ca90b779502b1898538a4d062a360fd64318a39b3761e9b3ea3004133da"
+                "sha256:7d68cfbcf7ba1e76671351949502c8107eaf0f300631e9d71929cffc58d82226",
+                "sha256:bd4d6cb69a8c718360f3d62552ef011c95539e5b2992d813b9200fad474bc0c5"
             ],
             "index": "ia",
-            "version": "==0.1.0"
+            "version": "==0.1.1"
         },
         "htmldate": {
             "hashes": [
@@ -1208,6 +1208,14 @@
             ],
             "version": "==0.2.0"
         },
+        "black": {
+            "hashes": [
+                "sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115",
+                "sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91"
+            ],
+            "index": "ia",
+            "version": "==21.9b0"
+        },
         "certifi": {
             "hashes": [
                 "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
@@ -1223,6 +1231,13 @@
             "markers": "python_version >= '3'",
             "version": "==2.0.7"
         },
+        "click": {
+            "hashes": [
+                "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3",
+                "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"
+            ],
+            "version": "==8.0.3"
+        },
         "coverage": {
             "extras": [
                 "toml"
@@ -1416,6 +1431,13 @@
             ],
             "version": "==0.8.2"
         },
+        "pathspec": {
+            "hashes": [
+                "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a",
+                "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"
+            ],
+            "version": "==0.9.0"
+        },
         "pexpect": {
             "hashes": [
                 "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
@@ -1497,10 +1519,10 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:84196357aa3566d64ad123d7a3c67b0e597a115c4934b097580e5ce220b91531",
-                "sha256:fd93fc45c47893c300bd98f5dd1b41c0e783eaeb727e7cea210dcc09d64ce7c3"
+                "sha256:9e3511118010f112a4b4b435ae50e1eaa610cda191acb9e421d60cf5fde83455",
+                "sha256:f8d3fe9fc404576c5164f0f0c4e382c96b85265e023c409c43d48f65da9d60d0"
             ],
-            "version": "==3.0.1"
+            "version": "==3.0.3"
         },
         "pytest": {
             "hashes": [
@@ -1541,6 +1563,47 @@
             "index": "ia",
             "version": "==0.7.3"
         },
+        "regex": {
+            "hashes": [
+                "sha256:0c186691a7995ef1db61205e00545bf161fb7b59cdb8c1201c89b333141c438a",
+                "sha256:0dcc0e71118be8c69252c207630faf13ca5e1b8583d57012aae191e7d6d28b84",
+                "sha256:0f7552429dd39f70057ac5d0e897e5bfe211629652399a21671e53f2a9693a4e",
+                "sha256:129472cd06062fb13e7b4670a102951a3e655e9b91634432cfbdb7810af9d710",
+                "sha256:13ec99df95003f56edcd307db44f06fbeb708c4ccdcf940478067dd62353181e",
+                "sha256:1f2b59c28afc53973d22e7bc18428721ee8ca6079becf1b36571c42627321c65",
+                "sha256:2b20f544cbbeffe171911f6ce90388ad36fe3fad26b7c7a35d4762817e9ea69c",
+                "sha256:2fb698037c35109d3c2e30f2beb499e5ebae6e4bb8ff2e60c50b9a805a716f79",
+                "sha256:34d870f9f27f2161709054d73646fc9aca49480617a65533fc2b4611c518e455",
+                "sha256:391703a2abf8013d95bae39145d26b4e21531ab82e22f26cd3a181ee2644c234",
+                "sha256:450dc27483548214314640c89a0f275dbc557968ed088da40bde7ef8fb52829e",
+                "sha256:45b65d6a275a478ac2cbd7fdbf7cc93c1982d613de4574b56fd6972ceadb8395",
+                "sha256:5095a411c8479e715784a0c9236568ae72509450ee2226b649083730f3fadfc6",
+                "sha256:530fc2bbb3dc1ebb17f70f7b234f90a1dd43b1b489ea38cea7be95fb21cdb5c7",
+                "sha256:56f0c81c44638dfd0e2367df1a331b4ddf2e771366c4b9c5d9a473de75e3e1c7",
+                "sha256:5e9c9e0ce92f27cef79e28e877c6b6988c48b16942258f3bc55d39b5f911df4f",
+                "sha256:6d7722136c6ed75caf84e1788df36397efdc5dbadab95e59c2bba82d4d808a4c",
+                "sha256:74d071dbe4b53c602edd87a7476ab23015a991374ddb228d941929ad7c8c922e",
+                "sha256:7b568809dca44cb75c8ebb260844ea98252c8c88396f9d203f5094e50a70355f",
+                "sha256:80bb5d2e92b2258188e7dcae5b188c7bf868eafdf800ea6edd0fbfc029984a88",
+                "sha256:8d1cdcda6bd16268316d5db1038965acf948f2a6f43acc2e0b1641ceab443623",
+                "sha256:9f665677e46c5a4d288ece12fdedf4f4204a422bb28ff05f0e6b08b7447796d1",
+                "sha256:a30513828180264294953cecd942202dfda64e85195ae36c265daf4052af0464",
+                "sha256:a7a986c45d1099a5de766a15de7bee3840b1e0e1a344430926af08e5297cf666",
+                "sha256:a940ca7e7189d23da2bfbb38973832813eab6bd83f3bf89a977668c2f813deae",
+                "sha256:ab7c5684ff3538b67df3f93d66bd3369b749087871ae3786e70ef39e601345b0",
+                "sha256:be04739a27be55631069b348dda0c81d8ea9822b5da10b8019b789e42d1fe452",
+                "sha256:c0938ddd60cc04e8f1faf7a14a166ac939aac703745bfcd8e8f20322a7373019",
+                "sha256:cb46b542133999580ffb691baf67410306833ee1e4f58ed06b6a7aaf4e046952",
+                "sha256:d134757a37d8640f3c0abb41f5e68b7cf66c644f54ef1cb0573b7ea1c63e1509",
+                "sha256:de557502c3bec8e634246588a94e82f1ee1b9dfcfdc453267c4fb652ff531570",
+                "sha256:ded0c4a3eee56b57fcb2315e40812b173cafe79d2f992d50015f4387445737fa",
+                "sha256:e1dae12321b31059a1a72aaa0e6ba30156fe7e633355e445451e4021b8e122b6",
+                "sha256:eb672217f7bd640411cfc69756ce721d00ae600814708d35c930930f18e8029f",
+                "sha256:ee684f139c91e69fe09b8e83d18b4d63bf87d9440c1eb2eeb52ee851883b1b29",
+                "sha256:f3f9a91d3cc5e5b0ddf1043c0ae5fa4852f18a1c0050318baf5fc7930ecc1f9c"
+            ],
+            "version": "==2021.10.23"
+        },
         "requests": {
             "hashes": [
                 "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
@@ -1551,11 +1614,11 @@
         },
         "responses": {
             "hashes": [
-                "sha256:57bab4e9d4d65f31ea5caf9de62095032c4d81f591a8fac2f5858f7777b8567b",
-                "sha256:93f774a762ee0e27c0d9d7e06227aeda9ff9f5f69392f72bb6c6b73f8763563e"
+                "sha256:5955ad3468fe8eb5fb736cdab4943457b7768f8670fa3624b4e26ff52dfe20c0",
+                "sha256:866757987d1962aa908d9c8b3185739faefd72a359e95459de0c2e4e5369c9b2"
             ],
             "index": "ia",
-            "version": "==0.14.0"
+            "version": "==0.15.0"
         },
         "six": {
             "hashes": [
@@ -1585,6 +1648,14 @@
             ],
             "version": "==5.1.1"
         },
+        "types-beautifulsoup4": {
+            "hashes": [
+                "sha256:116f5ca86b71c8667e65e4e6d1a03c2e991e14af6b14c2f79a939d5686f1d68a",
+                "sha256:5c191192c930b547ef72c6023b98f5dfbf2a032df59911db3e92cf3ef59df4b0"
+            ],
+            "index": "ia",
+            "version": "==4.10.4"
+        },
         "types-dateparser": {
             "hashes": [
                 "sha256:175f33965829f8393f0235799a71b2d102643c85e9cc704afe37b820b7062e96",
@@ -1593,6 +1664,22 @@
             "index": "ia",
             "version": "==1.0.7"
         },
+        "types-pillow": {
+            "hashes": [
+                "sha256:8f1d4eaedc0aa8c8fe4f912daff31abd79f04a42d9fda1b3f30c442f906fcff4",
+                "sha256:c76f76459e505b6621d2aaa66a69d3a4b123ccea3563d8aad200db1c16e8a8fa"
+            ],
+            "index": "ia",
+            "version": "==8.3.7"
+        },
+        "types-psycopg2": {
+            "hashes": [
+                "sha256:77ed80f2668582654623e04fb3d741ecce93effcc39c929d7e02f4a917a538ce",
+                "sha256:98a6e0e9580cd7eb4bd4d20f7c7063d154b2589a2b90c0ce4e3ca6085cde77c6"
+            ],
+            "index": "ia",
+            "version": "==2.9.1"
+        },
         "types-requests": {
             "hashes": [
                 "sha256:b279284e51f668e38ee12d9665e4d789089f532dc2a0be4a1508ca0efd98ba9e",
@@ -1672,14 +1759,6 @@
                 "sha256:fdede980273aeca591ad354608778365a3a310e0ecdd7a3587b38bc5be9b1808"
             ],
             "version": "==1.13.2"
-        },
-        "yapf": {
-            "hashes": [
-                "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d",
-                "sha256:e3a234ba8455fe201eaa649cdac872d590089a18b661e39bbac7020978dd9c2e"
-            ],
-            "index": "ia",
-            "version": "==0.31.0"
         }
     }
 }