diff options
Diffstat (limited to 'python')
54 files changed, 1835 insertions, 244 deletions
diff --git a/python/.gitignore b/python/.gitignore index e2dae299..a0bc258b 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,4 @@ +.env codegen-out/ build/ dist/ diff --git a/python/Pipfile b/python/Pipfile index 45052870..b968c2aa 100644 --- a/python/Pipfile +++ b/python/Pipfile @@ -18,15 +18,20 @@ pylint = "*" pg-view = "*" [packages] +python-dotenv = "*" Flask = "*" -requests = "*" -raven = { extras = ['flask'], version = "*" } +#Flask-OIDC = "*" flask-uuid = "*" flask-debugtoolbar = "*" +flask-login = "*" +loginpass = "*" +requests = "*" +raven = { extras = ['flask'], version = "*" } pykafka = "*" python-dateutil = "*" sickle = "*" python-snappy = "*" +pymacaroons = "*" [requires] # Python 3.5 is the bundled (system) version of python for Ubuntu 16.04 diff --git a/python/Pipfile.lock b/python/Pipfile.lock index 850e9848..1a370fc3 100644 --- a/python/Pipfile.lock +++ b/python/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "66bf3db374a8fafb8bb1e217d44ea993a809e71f6f65e2c42567ca588d1fe574" + "sha256": "6687e5eaeb0ca93b8051526e2c7ac844f3f0918d42f5701d25728745fce3925c" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,20 @@ ] }, "default": { + "asn1crypto": { + "hashes": [ + "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", + "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + ], + "version": "==0.24.0" + }, + "authlib": { + "hashes": [ + "sha256:b61c6c6fd230c4ba8602fd85ee9a40e6dc859387699a1cd1f7247c4b109dcc17", + "sha256:eda3e5af921a368091fef721d6d169bcff2aa0003d05113bc26e127f58c9a5e8" + ], + "version": "==0.10" + }, "blinker": { "hashes": [ "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" @@ -29,6 +43,43 @@ ], "version": "==2018.11.29" }, + "cffi": { + "hashes": [ + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + ], + "version": "==1.11.5" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -43,6 +94,30 @@ ], "version": "==7.0" }, + "cryptography": { + "hashes": [ + "sha256:05a6052c6a9f17ff78ba78f8e6eb1d777d25db3b763343a1ae89a7a8670386dd", + "sha256:0eb83a24c650a36f68e31a6d0a70f7ad9c358fa2506dc7b683398b92e354a038", + "sha256:0ff4a3d6ea86aa0c9e06e92a9f986de7ee8231f36c4da1b31c61a7e692ef3378", + "sha256:1699f3e916981df32afdd014fb3164db28cdb61c757029f502cb0a8c29b2fdb3", + "sha256:1b1f136d74f411f587b07c076149c4436a169dc19532e587460d9ced24adcc13", + "sha256:21e63dd20f5e5455e8b34179ac43d95b3fb1ffa54d071fd2ed5d67da82cfe6dc", + "sha256:2454ada8209bbde97065453a6ca488884bbb263e623d35ba183821317a58b46f", + "sha256:3cdc5f7ca057b2214ce4569e01b0f368b3de9d8ee01887557755ccd1c15d9427", + "sha256:418e7a5ec02a7056d3a4f0c0e7ea81df374205f25f4720bb0e84189aa5fd2515", + "sha256:471a097076a7c4ab85561d7fa9a1239bd2ae1f9fd0047520f13d8b340bf3210b", + "sha256:5ecaf9e7db3ca582c6de6229525d35db8a4e59dc3e8a40a331674ed90e658cbf", + "sha256:63b064a074f8dc61be81449796e2c3f4e308b6eba04a241a5c9f2d05e882c681", + "sha256:6afe324dfe6074822ccd56d80420df750e19ac30a4e56c925746c735cf22ae8b", + "sha256:70596e90398574b77929cd87e1ac6e43edd0e29ba01e1365fed9c26bde295aa5", + "sha256:70c2b04e905d3f72e2ba12c58a590817128dfca08949173faa19a42c824efa0b", + "sha256:8908f1db90be48b060888e9c96a0dee9d842765ce9594ff6a23da61086116bb6", + "sha256:af12dfc9874ac27ebe57fc28c8df0e8afa11f2a1025566476b0d50cdb8884f70", + "sha256:b4fc04326b2d259ddd59ed8ea20405d2e695486ab4c5e1e49b025c484845206e", + "sha256:da5b5dda4aa0d5e2b758cc8dfc67f8d4212e88ea9caad5f61ba132f948bab859" + ], + "version": "==2.4.2" + }, "flask": { "hashes": [ "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", @@ -59,6 +134,13 @@ "index": "pypi", "version": "==0.10.1" }, + "flask-login": { + "hashes": [ + "sha256:c815c1ac7b3e35e2081685e389a665f2c74d7e077cb93cecabaea352da4752ec" + ], + "index": "pypi", + "version": "==0.4.1" + }, "flask-uuid": { "hashes": [ "sha256:f9a8196eb896599ba9e74dcf713cfd1aca4669d418c19069e088620ae6294805" @@ -68,10 +150,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "itsdangerous": { "hashes": [ @@ -94,40 +176,48 @@ ], "version": "==2.5.0" }, + "loginpass": { + "hashes": [ + "sha256:0d87aa651ae6ff25194f4f7d8b85fdd780d356783f893b8921fe2ba5112aaf93", + "sha256:970e1debbd88c75cc5df693656fd86620817366108214f53d3af8edee09db428" + ], + "index": "pypi", + "version": "==0.2.1" + }, "lxml": { "hashes": [ - "sha256:02bc220d61f46e9b9d5a53c361ef95e9f5e1d27171cd461dddb17677ae2289a5", - "sha256:22f253b542a342755f6cfc047fe4d3a296515cf9b542bc6e261af45a80b8caf6", - "sha256:2f31145c7ff665b330919bfa44aacd3a0211a76ca7e7b441039d2a0b0451e415", - "sha256:36720698c29e7a9626a0dc802ef8885f8f0239bfd1689628ecd459a061f2807f", - "sha256:438a1b0203545521f6616132bfe0f4bca86f8a401364008b30e2b26ec408ce85", - "sha256:4815892904c336bbaf73dafd54f45f69f4021c22b5bad7332176bbf4fb830568", - "sha256:5be031b0f15ad63910d8e5038b489d95a79929513b3634ad4babf77100602588", - "sha256:5c93ae37c3c588e829b037fdfbd64a6e40c901d3f93f7beed6d724c44829a3ad", - "sha256:60842230678674cdac4a1cf0f707ef12d75b9a4fc4a565add4f710b5fcf185d5", - "sha256:62939a8bb6758d1bf923aa1c13f0bcfa9bf5b2fc0f5fa917a6e25db5fe0cfa4e", - "sha256:75830c06a62fe7b8fe3bbb5f269f0b308f19f3949ac81cfd40062f47c1455faf", - "sha256:81992565b74332c7c1aff6a913a3e906771aa81c9d0c68c68113cffcae45bc53", - "sha256:8c892fb0ee52c594d9a7751c7d7356056a9682674b92cc1c4dc968ff0f30c52f", - "sha256:9d862e3cf4fc1f2837dedce9c42269c8c76d027e49820a548ac89fdcee1e361f", - "sha256:a623965c086a6e91bb703d4da62dabe59fe88888e82c4117d544e11fd74835d6", - "sha256:a7783ab7f6a508b0510490cef9f857b763d796ba7476d9703f89722928d1e113", - "sha256:aab09fbe8abfa3b9ce62aaf45aca2d28726b1b9ee44871dbe644050a2fff4940", - "sha256:abf181934ac3ef193832fb973fd7f6149b5c531903c2ec0f1220941d73eee601", - "sha256:ae07fa0c115733fce1e9da96a3ac3fa24801742ca17e917e0c79d63a01eeb843", - "sha256:b9c78242219f674ab645ec571c9a95d70f381319a23911941cd2358a8e0521cf", - "sha256:bccb267678b870d9782c3b44d0cefe3ba0e329f9af8c946d32bf3778e7a4f271", - "sha256:c4df4d27f4c93b2cef74579f00b1d3a31a929c7d8023f870c4b476f03a274db4", - "sha256:caf0e50b546bb60dfa99bb18dfa6748458a83131ecdceaf5c071d74907e7e78a", - "sha256:d3266bd3ac59ac4edcd5fa75165dee80b94a3e5c91049df5f7c057ccf097551c", - "sha256:db0d213987bcd4e6d41710fb4532b22315b0d8fb439ff901782234456556aed1", - "sha256:dbbd5cf7690a40a9f0a9325ab480d0fccf46d16b378eefc08e195d84299bfae1", - "sha256:e16e07a0ec3a75b5ee61f2b1003c35696738f937dc8148fbda9fe2147ccb6e61", - "sha256:e175a006725c7faadbe69e791877d09936c0ef2cf49d01b60a6c1efcb0e8be6f", - "sha256:edd9c13a97f6550f9da2236126bb51c092b3b1ce6187f2bd966533ad794bbb5e", - "sha256:fa39ea60d527fbdd94215b5e5552f1c6a912624521093f1384a491a8ad89ad8b" - ], - "version": "==4.2.5" + "sha256:16cf8bac33ec17049617186d63006ba49da7c5be417042877a49f0ef6d7a195d", + "sha256:18f2d8f14cc61e66e8a45f740d15b6fc683c096f733db1f8d0ee15bcac9843de", + "sha256:260868f69d14a64dd1de9cf92e133d2f71514d288de4906f109bdf48ca9b756a", + "sha256:29b8acd8ecdf772266dbac491f203c71664b0b07ad4309ba2c3bb131306332fc", + "sha256:2b05e5e06f8e8c63595472dc887d0d6e0250af754a35ba690f6a6abf2ef85691", + "sha256:30d6ec05fb607a5b7345549f642c7c7a5b747b634f6d5e935596b910f243f96f", + "sha256:3bf683f0237449ebc1851098f664410e3c99ba3faa8c9cc82c6acfe857df1767", + "sha256:3ce5488121eb15513c4b239dadd67f9e7959511bd766aac6be0c35e80274f298", + "sha256:48be0c375350a5519bb9474b42a9c0e7ab709fb45f11bfcd33de876791137896", + "sha256:49bc343ca3b30cd860845433bb9f62448a54ff87b632175108bacbc5dc63e49e", + "sha256:4cc7531e86a43ea66601763c5914c3d3adb297f32e4284957609b90d41825fca", + "sha256:4e9822fad564d82035f0b6d701a890444560210f8a8648b8f15850f8fe883cd9", + "sha256:51a9a441aefc8c93512bad5efe867d2ff086e7249ce0fc3b47c310644b352936", + "sha256:5bbed9efc8aeb69929140f71a30e655bf496b45b766861513960e1b11168d475", + "sha256:60a5323b2bc893ca1059d283d6695a172d51cc95a70c25b3e587e1aad5459c38", + "sha256:7035d9361f3ceec9ccc1dd3482094d1174580e7e1bf6870b77ea758f7cad15d2", + "sha256:76d62cc048bda0ebf476689ad3eb8e65e6827e43a7521be3b163071020667b8c", + "sha256:78163b578e6d1836012febaa1865e095ccc7fc826964dd69a2dbfe401618a1f7", + "sha256:83b58b2b5904d50de03a47e2f56d24e9da4cf7e3b0d66fb4510b18fca0faf910", + "sha256:a07447e46fffa5bb4d7a0af0a6505c8517e9bd197cfd2aec79e499b6e86cde49", + "sha256:a17d808b3edca4aaf6b295b5a388c844a0b7f79aca2d79eec5acc1461db739e3", + "sha256:a378fd61022cf4d3b492134c3bc48204ac2ff19e0813b23e07c3dd95ae8df0bc", + "sha256:aa7d096a44ae3d475c5ed763e24cf302d32462e78b61bba73ce1ad0efb8f522a", + "sha256:ade8785c93a985956ba6499d5ea6d0a362e24b4a9ba07dd18920fd67cccf63ea", + "sha256:cc039668f91d8af8c4094cfb5a67c7ae733967fdc84c0507fe271db81480d367", + "sha256:d89f1ffe98744c4b5c11f00fb843a4e72f68a6279b5e38168167f1b3c0fdd84c", + "sha256:e691b6ef6e27437860016bd6c32e481bdc2ed3af03289707a38b9ca422105f40", + "sha256:e750da6ac3ca624ae3303df448664012f9b6f9dfbc5d50048ea8a12ce2f8bc29", + "sha256:eca305b200549906ea25648463aeb1b3b220b716415183eaa99c998a846936d9", + "sha256:f52fe795e08858192eea167290033b5ff24f50f51781cb78d989e8d63cfe73d1" + ], + "version": "==4.2.6" }, "markupsafe": { "hashes": [ @@ -162,6 +252,12 @@ ], "version": "==1.1.0" }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, "pykafka": { "hashes": [ "sha256:6b075909a52cb0c95325bc16ab797bbcdbb37386652ea460705ed4472ce91459", @@ -170,6 +266,38 @@ "index": "pypi", "version": "==2.8.0" }, + "pymacaroons": { + "hashes": [ + "sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8", + "sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907" + ], + "index": "pypi", + "version": "==0.13.0" + }, + "pynacl": { + "hashes": [ + "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", + "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", + "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", + "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", + "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", + "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", + "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", + "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", + "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", + "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", + "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", + "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", + "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", + "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", + "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", + "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", + "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + ], + "version": "==1.3.0" + }, "python-dateutil": { "hashes": [ "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", @@ -178,6 +306,14 @@ "index": "pypi", "version": "==2.7.5" }, + "python-dotenv": { + "hashes": [ + "sha256:a84569d0e00d178bc5b957f7ff208bf49287cbf61857c31c258c4a91f571527b", + "sha256:c9b1ddd3cdbe75c7d462cb84674d87130f4b948f090f02c7d7144779afb99ae0" + ], + "index": "pypi", + "version": "==0.10.1" + }, "python-snappy": { "hashes": [ "sha256:59c79d83350f931ad5cf8f06ccb1c9bd1087a77c3ca7e00806884cda654a6faf", @@ -188,19 +324,19 @@ }, "raven": { "hashes": [ - "sha256:3fd787d19ebb49919268f06f19310e8112d619ef364f7989246fc8753d469888", - "sha256:95f44f3ea2c1b176d5450df4becdb96c15bf2632888f9ab193e9dd22300ce46a" + "sha256:3fa6de6efa2493a7c827472e984ce9b020797d0da16f1db67197bcc23c8fae54", + "sha256:44a13f87670836e153951af9a3c80405d36b43097db869a36e92809673692ce4" ], "index": "pypi", - "version": "==6.9.0" + "version": "==6.10.0" }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], "index": "pypi", - "version": "==2.20.1" + "version": "==2.21.0" }, "sickle": { "hashes": [ @@ -336,10 +472,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "ipython": { "hashes": [ @@ -366,10 +502,10 @@ }, "jedi": { "hashes": [ - "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7", - "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148" + "sha256:571702b5bd167911fe9036e5039ba67f820d6502832285cde8c881ab2b2149fd", + "sha256:c8481b5e59d34a5c7c42e98f6625e633f6ef59353abea6437472c7ec2093f191" ], - "version": "==0.13.1" + "version": "==0.13.2" }, "lazy-object-proxy": { "hashes": [ @@ -414,11 +550,11 @@ }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", + "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", + "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" ], - "version": "==4.3.0" + "version": "==5.0.0" }, "parso": { "hashes": [ @@ -523,10 +659,10 @@ }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", + "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pylint": { "hashes": [ @@ -538,11 +674,11 @@ }, "pytest": { "hashes": [ - "sha256:1d131cc532be0023ef8ae265e2a779938d0619bb6c2510f52987ffcba7fa1ee4", - "sha256:ca4761407f1acc85ffd1609f464ca20bb71a767803505bd4127d0e45c5a50e23" + "sha256:f689bf2fc18c4585403348dd56f47d87780bf217c53ed9ae7a3e2d7faa45f8e9", + "sha256:f812ea39a0153566be53d88f8de94839db1e8a05352ed8a49525d7d7f37861e9" ], "index": "pypi", - "version": "==4.0.1" + "version": "==4.0.2" }, "pytest-cov": { "hashes": [ @@ -561,19 +697,19 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], "index": "pypi", - "version": "==2.20.1" + "version": "==2.21.0" }, "responses": { "hashes": [ - "sha256:16ad4a7a914f20792111157adf09c63a8dc37699c57d1ad20dbc281a4f5743fb", - "sha256:b9b31d9b1fcf6d48aea044c9fdd3d04199f6d227b0650c15d2566b0135bc1ed7" + "sha256:c85882d2dc608ce6b5713a4e1534120f4a0dc6ec79d1366570d2b0c909a50c87", + "sha256:ea5a14f9aea173e3b786ff04cf03133c2dabd4103dbaef1028742fd71a6c2ad3" ], "index": "pypi", - "version": "==0.10.4" + "version": "==0.10.5" }, "six": { "hashes": [ @@ -591,32 +727,30 @@ }, "typed-ast": { "hashes": [ - "sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58", - "sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d", - "sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291", - "sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a", - "sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9", - "sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892", - "sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9", - "sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded", - "sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa", - "sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe", - "sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd", - "sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85", - "sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6", - "sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46", - "sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51", - "sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f", - "sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129", - "sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c", - "sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea", - "sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863", - "sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559", - "sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87", - "sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6" + "sha256:0555eca1671ebe09eb5f2176723826f6f44cca5060502fea259de9b0e893ab53", + "sha256:0ca96128ea66163aea13911c9b4b661cb345eb729a20be15c034271360fc7474", + "sha256:16ccd06d614cf81b96de42a37679af12526ea25a208bce3da2d9226f44563868", + "sha256:1e21ae7b49a3f744958ffad1737dfbdb43e1137503ccc59f4e32c4ac33b0bd1c", + "sha256:37670c6fd857b5eb68aa5d193e14098354783b5138de482afa401cc2644f5a7f", + "sha256:46d84c8e3806619ece595aaf4f37743083f9454c9ea68a517f1daa05126daf1d", + "sha256:5b972bbb3819ece283a67358103cc6671da3646397b06e7acea558444daf54b2", + "sha256:6306ffa64922a7b58ee2e8d6f207813460ca5a90213b4a400c2e730375049246", + "sha256:6cb25dc95078931ecbd6cbcc4178d1b8ae8f2b513ae9c3bd0b7f81c2191db4c6", + "sha256:7e19d439fee23620dea6468d85bfe529b873dace39b7e5b0c82c7099681f8a22", + "sha256:7f5cd83af6b3ca9757e1127d852f497d11c7b09b4716c355acfbebf783d028da", + "sha256:81e885a713e06faeef37223a5b1167615db87f947ecc73f815b9d1bbd6b585be", + "sha256:94af325c9fe354019a29f9016277c547ad5d8a2d98a02806f27a7436b2da6735", + "sha256:b1e5445c6075f509d5764b84ce641a1535748801253b97f3b7ea9d948a22853a", + "sha256:cb061a959fec9a514d243831c514b51ccb940b58a5ce572a4e209810f2507dcf", + "sha256:cc8d0b703d573cbabe0d51c9d68ab68df42a81409e4ed6af45a04a95484b96a5", + "sha256:da0afa955865920edb146926455ec49da20965389982f91e926389666f5cf86a", + "sha256:dc76738331d61818ce0b90647aedde17bbba3d3f9e969d83c1d9087b4f978862", + "sha256:e7ec9a1445d27dbd0446568035f7106fa899a36f55e52ade28020f7b3845180d", + "sha256:f741ba03feb480061ab91a465d1a3ed2d40b52822ada5b4017770dfcb88f839f", + "sha256:fe800a58547dd424cd286b7270b967b5b3316b993d86453ede184a17b5a6b17d" ], "markers": "python_version < '3.7' and implementation_name == 'cpython'", - "version": "==1.1.0" + "version": "==1.1.1" }, "urllib3": { "hashes": [ diff --git a/python/README.md b/python/README.md index d8812934..922499f3 100644 --- a/python/README.md +++ b/python/README.md @@ -34,6 +34,10 @@ server on the same machine by default), use: # will listen on http://localhost:9810 by default pipenv run fatcat_webface.py +Almost all configuration is done via environment variables; see `env.example` +for a list of settings. If you copy this file to `.env` it will be sourced by +`pipenv` automatically; you can also load it in your shell like `source .env`. + ## Running Tests Many (though not all) python tests depend on access to a local running API diff --git a/python/env.example b/python/env.example new file mode 100644 index 00000000..a6935de9 --- /dev/null +++ b/python/env.example @@ -0,0 +1,19 @@ +FLASK_SECRET_KEY="" +# This key used in tests +FATCAT_API_AUTH_TOKEN="AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFpAAIbdGltZSA+IDIwMTktMDEtMDhUMjM6MzY6NDRaAAAGIK1s5s0Z75h6yqaVa5b9grvEKBBE4pVnBieOc0CaKTI1" +FATCAT_API_HOST="http://localhost:9411/v0" +ELASTICSEARCH_BACKEND="http://localhost:9200" +ELASTICSEARCH_INDEX="fatcat" +GITLAB_CLIENT_ID="" +GITLAB_CLIENT_SECRET="" +IA_XAUTH_CLIENT_ID="" +IA_XAUTH_CLIENT_SECRET="" +SENTRY_DSN="" + +# These auth keys only for workers/importers; locally will fall back to +# FATCAT_API_AUTH_TOKEN +FATCAT_AUTH_WORKER_CROSSREF="" +FATCAT_AUTH_WORKER_ORCID="" +FATCAT_AUTH_WORKER_ISSN="" +FATCAT_AUTH_WORKER_MATCHED="" +FATCAT_AUTH_WORKER_GROBID_METADATA="" diff --git a/python/fatcat_client/README.md b/python/fatcat_client/README.md index 0f170925..8704641e 100644 --- a/python/fatcat_client/README.md +++ b/python/fatcat_client/README.md @@ -50,6 +50,11 @@ import time import fatcat_client from fatcat_client.rest import ApiException from pprint import pprint + +# Configure API key authorization: Bearer +fatcat_client.configuration.api_key['Authorization'] = 'YOUR_API_KEY' +# Uncomment below to setup prefix (e.g. Bearer) for API key, if needed +# fatcat_client.configuration.api_key_prefix['Authorization'] = 'Bearer' # create an instance of the API class api_instance = fatcat_client.DefaultApi() editgroup_id = 'editgroup_id_example' # str | base32-encoded unique identifier @@ -69,6 +74,8 @@ All URIs are relative to *https://api.fatcat.wiki/v0* Class | Method | HTTP request | Description ------------ | ------------- | ------------- | ------------- *DefaultApi* | [**accept_editgroup**](docs/DefaultApi.md#accept_editgroup) | **POST** /editgroup/{editgroup_id}/accept | +*DefaultApi* | [**auth_check**](docs/DefaultApi.md#auth_check) | **GET** /auth/check | +*DefaultApi* | [**auth_oidc**](docs/DefaultApi.md#auth_oidc) | **POST** /auth/oidc | *DefaultApi* | [**create_container**](docs/DefaultApi.md#create_container) | **POST** /container | *DefaultApi* | [**create_container_batch**](docs/DefaultApi.md#create_container_batch) | **POST** /container/batch | *DefaultApi* | [**create_creator**](docs/DefaultApi.md#create_creator) | **POST** /creator | @@ -149,6 +156,7 @@ Class | Method | HTTP request | Description *DefaultApi* | [**lookup_release**](docs/DefaultApi.md#lookup_release) | **GET** /release/lookup | *DefaultApi* | [**update_container**](docs/DefaultApi.md#update_container) | **PUT** /container/{ident} | *DefaultApi* | [**update_creator**](docs/DefaultApi.md#update_creator) | **PUT** /creator/{ident} | +*DefaultApi* | [**update_editor**](docs/DefaultApi.md#update_editor) | **PUT** /editor/{editor_id} | *DefaultApi* | [**update_file**](docs/DefaultApi.md#update_file) | **PUT** /file/{ident} | *DefaultApi* | [**update_fileset**](docs/DefaultApi.md#update_fileset) | **PUT** /fileset/{ident} | *DefaultApi* | [**update_release**](docs/DefaultApi.md#update_release) | **PUT** /release/{ident} | @@ -158,6 +166,8 @@ Class | Method | HTTP request | Description ## Documentation For Models + - [AuthOidc](docs/AuthOidc.md) + - [AuthOidcResult](docs/AuthOidcResult.md) - [ChangelogEntry](docs/ChangelogEntry.md) - [ContainerEntity](docs/ContainerEntity.md) - [CreatorEntity](docs/CreatorEntity.md) @@ -184,7 +194,12 @@ Class | Method | HTTP request | Description ## Documentation For Authorization - All endpoints do not require authorization. + +## Bearer + +- **Type**: API key +- **API key parameter name**: Authorization +- **Location**: HTTP header ## Author diff --git a/python/fatcat_client/__init__.py b/python/fatcat_client/__init__.py index 6b08c0b1..e9a04942 100644 --- a/python/fatcat_client/__init__.py +++ b/python/fatcat_client/__init__.py @@ -22,6 +22,8 @@ from fatcat_client.api.default_api import DefaultApi from fatcat_client.api_client import ApiClient from fatcat_client.configuration import Configuration # import models into sdk package +from fatcat_client.models.auth_oidc import AuthOidc +from fatcat_client.models.auth_oidc_result import AuthOidcResult from fatcat_client.models.changelog_entry import ChangelogEntry from fatcat_client.models.container_entity import ContainerEntity from fatcat_client.models.creator_entity import CreatorEntity diff --git a/python/fatcat_client/api/default_api.py b/python/fatcat_client/api/default_api.py index 9f7edf07..8b652571 100644 --- a/python/fatcat_client/api/default_api.py +++ b/python/fatcat_client/api/default_api.py @@ -120,7 +120,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/editgroup/{editgroup_id}/accept', 'POST', @@ -138,6 +138,196 @@ class DefaultApi(object): _request_timeout=params.get('_request_timeout'), collection_formats=collection_formats) + def auth_check(self, **kwargs): # noqa: E501 + """auth_check # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.auth_check(async=True) + >>> result = thread.get() + + :param async bool + :param str role: + :return: Success + If the method is called asynchronously, + returns the request thread. + """ + kwargs['_return_http_data_only'] = True + if kwargs.get('async'): + return self.auth_check_with_http_info(**kwargs) # noqa: E501 + else: + (data) = self.auth_check_with_http_info(**kwargs) # noqa: E501 + return data + + def auth_check_with_http_info(self, **kwargs): # noqa: E501 + """auth_check # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.auth_check_with_http_info(async=True) + >>> result = thread.get() + + :param async bool + :param str role: + :return: Success + If the method is called asynchronously, + returns the request thread. + """ + + all_params = ['role'] # noqa: E501 + all_params.append('async') + all_params.append('_return_http_data_only') + all_params.append('_preload_content') + all_params.append('_request_timeout') + + params = locals() + for key, val in six.iteritems(params['kwargs']): + if key not in all_params: + raise TypeError( + "Got an unexpected keyword argument '%s'" + " to method auth_check" % key + ) + params[key] = val + del params['kwargs'] + + collection_formats = {} + + path_params = {} + + query_params = [] + if 'role' in params: + query_params.append(('role', params['role'])) # noqa: E501 + + header_params = {} + + form_params = [] + local_var_files = {} + + body_params = None + # HTTP header `Accept` + header_params['Accept'] = self.api_client.select_header_accept( + ['application/json']) # noqa: E501 + + # HTTP header `Content-Type` + header_params['Content-Type'] = self.api_client.select_header_content_type( # noqa: E501 + ['application/json']) # noqa: E501 + + # Authentication setting + auth_settings = ['Bearer'] # noqa: E501 + + return self.api_client.call_api( + '/auth/check', 'GET', + path_params, + query_params, + header_params, + body=body_params, + post_params=form_params, + files=local_var_files, + response_type='Success', # noqa: E501 + auth_settings=auth_settings, + async=params.get('async'), + _return_http_data_only=params.get('_return_http_data_only'), + _preload_content=params.get('_preload_content', True), + _request_timeout=params.get('_request_timeout'), + collection_formats=collection_formats) + + def auth_oidc(self, oidc_params, **kwargs): # noqa: E501 + """auth_oidc # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.auth_oidc(oidc_params, async=True) + >>> result = thread.get() + + :param async bool + :param AuthOidc oidc_params: (required) + :return: AuthOidcResult + If the method is called asynchronously, + returns the request thread. + """ + kwargs['_return_http_data_only'] = True + if kwargs.get('async'): + return self.auth_oidc_with_http_info(oidc_params, **kwargs) # noqa: E501 + else: + (data) = self.auth_oidc_with_http_info(oidc_params, **kwargs) # noqa: E501 + return data + + def auth_oidc_with_http_info(self, oidc_params, **kwargs): # noqa: E501 + """auth_oidc # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.auth_oidc_with_http_info(oidc_params, async=True) + >>> result = thread.get() + + :param async bool + :param AuthOidc oidc_params: (required) + :return: AuthOidcResult + If the method is called asynchronously, + returns the request thread. + """ + + all_params = ['oidc_params'] # noqa: E501 + all_params.append('async') + all_params.append('_return_http_data_only') + all_params.append('_preload_content') + all_params.append('_request_timeout') + + params = locals() + for key, val in six.iteritems(params['kwargs']): + if key not in all_params: + raise TypeError( + "Got an unexpected keyword argument '%s'" + " to method auth_oidc" % key + ) + params[key] = val + del params['kwargs'] + # verify the required parameter 'oidc_params' is set + if ('oidc_params' not in params or + params['oidc_params'] is None): + raise ValueError("Missing the required parameter `oidc_params` when calling `auth_oidc`") # noqa: E501 + + collection_formats = {} + + path_params = {} + + query_params = [] + + header_params = {} + + form_params = [] + local_var_files = {} + + body_params = None + if 'oidc_params' in params: + body_params = params['oidc_params'] + # HTTP header `Accept` + header_params['Accept'] = self.api_client.select_header_accept( + ['application/json']) # noqa: E501 + + # HTTP header `Content-Type` + header_params['Content-Type'] = self.api_client.select_header_content_type( # noqa: E501 + ['application/json']) # noqa: E501 + + # Authentication setting + auth_settings = ['Bearer'] # noqa: E501 + + return self.api_client.call_api( + '/auth/oidc', 'POST', + path_params, + query_params, + header_params, + body=body_params, + post_params=form_params, + files=local_var_files, + response_type='AuthOidcResult', # noqa: E501 + auth_settings=auth_settings, + async=params.get('async'), + _return_http_data_only=params.get('_return_http_data_only'), + _preload_content=params.get('_preload_content', True), + _request_timeout=params.get('_request_timeout'), + collection_formats=collection_formats) + def create_container(self, entity, **kwargs): # noqa: E501 """create_container # noqa: E501 @@ -221,7 +411,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/container', 'POST', @@ -326,7 +516,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/container/batch', 'POST', @@ -427,7 +617,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/creator', 'POST', @@ -532,7 +722,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/creator/batch', 'POST', @@ -629,7 +819,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/editgroup', 'POST', @@ -730,7 +920,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/file', 'POST', @@ -835,7 +1025,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/file/batch', 'POST', @@ -936,7 +1126,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/fileset', 'POST', @@ -1041,7 +1231,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/fileset/batch', 'POST', @@ -1142,7 +1332,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/release', 'POST', @@ -1247,7 +1437,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/release/batch', 'POST', @@ -1348,7 +1538,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/webcapture', 'POST', @@ -1453,7 +1643,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/webcapture/batch', 'POST', @@ -1554,7 +1744,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/work', 'POST', @@ -1659,7 +1849,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/work/batch', 'POST', @@ -1760,7 +1950,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/container/{ident}', 'DELETE', @@ -1865,7 +2055,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/container/edit/{edit_id}', 'DELETE', @@ -1966,7 +2156,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/creator/{ident}', 'DELETE', @@ -2071,7 +2261,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/creator/edit/{edit_id}', 'DELETE', @@ -2172,7 +2362,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/file/{ident}', 'DELETE', @@ -2277,7 +2467,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/file/edit/{edit_id}', 'DELETE', @@ -2378,7 +2568,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/fileset/{ident}', 'DELETE', @@ -2483,7 +2673,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/fileset/edit/{edit_id}', 'DELETE', @@ -2584,7 +2774,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/release/{ident}', 'DELETE', @@ -2689,7 +2879,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/release/edit/{edit_id}', 'DELETE', @@ -2790,7 +2980,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/webcapture/{ident}', 'DELETE', @@ -2895,7 +3085,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/webcapture/edit/{edit_id}', 'DELETE', @@ -2996,7 +3186,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/work/{ident}', 'DELETE', @@ -3101,7 +3291,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/work/edit/{edit_id}', 'DELETE', @@ -8331,7 +8521,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/container/{ident}', 'PUT', @@ -8440,7 +8630,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/creator/{ident}', 'PUT', @@ -8458,6 +8648,111 @@ class DefaultApi(object): _request_timeout=params.get('_request_timeout'), collection_formats=collection_formats) + def update_editor(self, editor_id, editor, **kwargs): # noqa: E501 + """update_editor # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.update_editor(editor_id, editor, async=True) + >>> result = thread.get() + + :param async bool + :param str editor_id: (required) + :param Editor editor: (required) + :return: Editor + If the method is called asynchronously, + returns the request thread. + """ + kwargs['_return_http_data_only'] = True + if kwargs.get('async'): + return self.update_editor_with_http_info(editor_id, editor, **kwargs) # noqa: E501 + else: + (data) = self.update_editor_with_http_info(editor_id, editor, **kwargs) # noqa: E501 + return data + + def update_editor_with_http_info(self, editor_id, editor, **kwargs): # noqa: E501 + """update_editor # noqa: E501 + + This method makes a synchronous HTTP request by default. To make an + asynchronous HTTP request, please pass async=True + >>> thread = api.update_editor_with_http_info(editor_id, editor, async=True) + >>> result = thread.get() + + :param async bool + :param str editor_id: (required) + :param Editor editor: (required) + :return: Editor + If the method is called asynchronously, + returns the request thread. + """ + + all_params = ['editor_id', 'editor'] # noqa: E501 + all_params.append('async') + all_params.append('_return_http_data_only') + all_params.append('_preload_content') + all_params.append('_request_timeout') + + params = locals() + for key, val in six.iteritems(params['kwargs']): + if key not in all_params: + raise TypeError( + "Got an unexpected keyword argument '%s'" + " to method update_editor" % key + ) + params[key] = val + del params['kwargs'] + # verify the required parameter 'editor_id' is set + if ('editor_id' not in params or + params['editor_id'] is None): + raise ValueError("Missing the required parameter `editor_id` when calling `update_editor`") # noqa: E501 + # verify the required parameter 'editor' is set + if ('editor' not in params or + params['editor'] is None): + raise ValueError("Missing the required parameter `editor` when calling `update_editor`") # noqa: E501 + + collection_formats = {} + + path_params = {} + if 'editor_id' in params: + path_params['editor_id'] = params['editor_id'] # noqa: E501 + + query_params = [] + + header_params = {} + + form_params = [] + local_var_files = {} + + body_params = None + if 'editor' in params: + body_params = params['editor'] + # HTTP header `Accept` + header_params['Accept'] = self.api_client.select_header_accept( + ['application/json']) # noqa: E501 + + # HTTP header `Content-Type` + header_params['Content-Type'] = self.api_client.select_header_content_type( # noqa: E501 + ['application/json']) # noqa: E501 + + # Authentication setting + auth_settings = ['Bearer'] # noqa: E501 + + return self.api_client.call_api( + '/editor/{editor_id}', 'PUT', + path_params, + query_params, + header_params, + body=body_params, + post_params=form_params, + files=local_var_files, + response_type='Editor', # noqa: E501 + auth_settings=auth_settings, + async=params.get('async'), + _return_http_data_only=params.get('_return_http_data_only'), + _preload_content=params.get('_preload_content', True), + _request_timeout=params.get('_request_timeout'), + collection_formats=collection_formats) + def update_file(self, ident, entity, **kwargs): # noqa: E501 """update_file # noqa: E501 @@ -8549,7 +8844,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/file/{ident}', 'PUT', @@ -8658,7 +8953,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/fileset/{ident}', 'PUT', @@ -8767,7 +9062,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/release/{ident}', 'PUT', @@ -8876,7 +9171,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/webcapture/{ident}', 'PUT', @@ -8985,7 +9280,7 @@ class DefaultApi(object): ['application/json']) # noqa: E501 # Authentication setting - auth_settings = [] # noqa: E501 + auth_settings = ['Bearer'] # noqa: E501 return self.api_client.call_api( '/work/{ident}', 'PUT', diff --git a/python/fatcat_client/configuration.py b/python/fatcat_client/configuration.py index 1dc47841..69b54edb 100644 --- a/python/fatcat_client/configuration.py +++ b/python/fatcat_client/configuration.py @@ -224,6 +224,13 @@ class Configuration(six.with_metaclass(TypeWithDefault, object)): :return: The Auth Settings information dict. """ return { + 'Bearer': + { + 'type': 'api_key', + 'in': 'header', + 'key': 'Authorization', + 'value': self.get_api_key_with_prefix('Authorization') + }, } diff --git a/python/fatcat_client/models/__init__.py b/python/fatcat_client/models/__init__.py index f4716d61..ef77b4fd 100644 --- a/python/fatcat_client/models/__init__.py +++ b/python/fatcat_client/models/__init__.py @@ -15,6 +15,8 @@ from __future__ import absolute_import # import models into model package +from fatcat_client.models.auth_oidc import AuthOidc +from fatcat_client.models.auth_oidc_result import AuthOidcResult from fatcat_client.models.changelog_entry import ChangelogEntry from fatcat_client.models.container_entity import ContainerEntity from fatcat_client.models.creator_entity import CreatorEntity diff --git a/python/fatcat_client/models/auth_oidc.py b/python/fatcat_client/models/auth_oidc.py new file mode 100644 index 00000000..1ee4c429 --- /dev/null +++ b/python/fatcat_client/models/auth_oidc.py @@ -0,0 +1,194 @@ +# coding: utf-8 + +""" + fatcat + + A scalable, versioned, API-oriented catalog of bibliographic entities and file metadata # noqa: E501 + + OpenAPI spec version: 0.1.0 + + Generated by: https://github.com/swagger-api/swagger-codegen.git +""" + + +import pprint +import re # noqa: F401 + +import six + + +class AuthOidc(object): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + + """ + Attributes: + swagger_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + swagger_types = { + 'provider': 'str', + 'sub': 'str', + 'iss': 'str', + 'preferred_username': 'str' + } + + attribute_map = { + 'provider': 'provider', + 'sub': 'sub', + 'iss': 'iss', + 'preferred_username': 'preferred_username' + } + + def __init__(self, provider=None, sub=None, iss=None, preferred_username=None): # noqa: E501 + """AuthOidc - a model defined in Swagger""" # noqa: E501 + + self._provider = None + self._sub = None + self._iss = None + self._preferred_username = None + self.discriminator = None + + self.provider = provider + self.sub = sub + self.iss = iss + self.preferred_username = preferred_username + + @property + def provider(self): + """Gets the provider of this AuthOidc. # noqa: E501 + + + :return: The provider of this AuthOidc. # noqa: E501 + :rtype: str + """ + return self._provider + + @provider.setter + def provider(self, provider): + """Sets the provider of this AuthOidc. + + + :param provider: The provider of this AuthOidc. # noqa: E501 + :type: str + """ + if provider is None: + raise ValueError("Invalid value for `provider`, must not be `None`") # noqa: E501 + + self._provider = provider + + @property + def sub(self): + """Gets the sub of this AuthOidc. # noqa: E501 + + + :return: The sub of this AuthOidc. # noqa: E501 + :rtype: str + """ + return self._sub + + @sub.setter + def sub(self, sub): + """Sets the sub of this AuthOidc. + + + :param sub: The sub of this AuthOidc. # noqa: E501 + :type: str + """ + if sub is None: + raise ValueError("Invalid value for `sub`, must not be `None`") # noqa: E501 + + self._sub = sub + + @property + def iss(self): + """Gets the iss of this AuthOidc. # noqa: E501 + + + :return: The iss of this AuthOidc. # noqa: E501 + :rtype: str + """ + return self._iss + + @iss.setter + def iss(self, iss): + """Sets the iss of this AuthOidc. + + + :param iss: The iss of this AuthOidc. # noqa: E501 + :type: str + """ + if iss is None: + raise ValueError("Invalid value for `iss`, must not be `None`") # noqa: E501 + + self._iss = iss + + @property + def preferred_username(self): + """Gets the preferred_username of this AuthOidc. # noqa: E501 + + + :return: The preferred_username of this AuthOidc. # noqa: E501 + :rtype: str + """ + return self._preferred_username + + @preferred_username.setter + def preferred_username(self, preferred_username): + """Sets the preferred_username of this AuthOidc. + + + :param preferred_username: The preferred_username of this AuthOidc. # noqa: E501 + :type: str + """ + if preferred_username is None: + raise ValueError("Invalid value for `preferred_username`, must not be `None`") # noqa: E501 + + self._preferred_username = preferred_username + + def to_dict(self): + """Returns the model properties as a dict""" + result = {} + + for attr, _ in six.iteritems(self.swagger_types): + value = getattr(self, attr) + if isinstance(value, list): + result[attr] = list(map( + lambda x: x.to_dict() if hasattr(x, "to_dict") else x, + value + )) + elif hasattr(value, "to_dict"): + result[attr] = value.to_dict() + elif isinstance(value, dict): + result[attr] = dict(map( + lambda item: (item[0], item[1].to_dict()) + if hasattr(item[1], "to_dict") else item, + value.items() + )) + else: + result[attr] = value + + return result + + def to_str(self): + """Returns the string representation of the model""" + return pprint.pformat(self.to_dict()) + + def __repr__(self): + """For `print` and `pprint`""" + return self.to_str() + + def __eq__(self, other): + """Returns true if both objects are equal""" + if not isinstance(other, AuthOidc): + return False + + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + """Returns true if both objects are not equal""" + return not self == other diff --git a/python/fatcat_client/models/auth_oidc_result.py b/python/fatcat_client/models/auth_oidc_result.py new file mode 100644 index 00000000..5e31a574 --- /dev/null +++ b/python/fatcat_client/models/auth_oidc_result.py @@ -0,0 +1,142 @@ +# coding: utf-8 + +""" + fatcat + + A scalable, versioned, API-oriented catalog of bibliographic entities and file metadata # noqa: E501 + + OpenAPI spec version: 0.1.0 + + Generated by: https://github.com/swagger-api/swagger-codegen.git +""" + + +import pprint +import re # noqa: F401 + +import six + +from fatcat_client.models.editor import Editor # noqa: F401,E501 + + +class AuthOidcResult(object): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + + """ + Attributes: + swagger_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + swagger_types = { + 'editor': 'Editor', + 'token': 'str' + } + + attribute_map = { + 'editor': 'editor', + 'token': 'token' + } + + def __init__(self, editor=None, token=None): # noqa: E501 + """AuthOidcResult - a model defined in Swagger""" # noqa: E501 + + self._editor = None + self._token = None + self.discriminator = None + + self.editor = editor + self.token = token + + @property + def editor(self): + """Gets the editor of this AuthOidcResult. # noqa: E501 + + + :return: The editor of this AuthOidcResult. # noqa: E501 + :rtype: Editor + """ + return self._editor + + @editor.setter + def editor(self, editor): + """Sets the editor of this AuthOidcResult. + + + :param editor: The editor of this AuthOidcResult. # noqa: E501 + :type: Editor + """ + if editor is None: + raise ValueError("Invalid value for `editor`, must not be `None`") # noqa: E501 + + self._editor = editor + + @property + def token(self): + """Gets the token of this AuthOidcResult. # noqa: E501 + + + :return: The token of this AuthOidcResult. # noqa: E501 + :rtype: str + """ + return self._token + + @token.setter + def token(self, token): + """Sets the token of this AuthOidcResult. + + + :param token: The token of this AuthOidcResult. # noqa: E501 + :type: str + """ + if token is None: + raise ValueError("Invalid value for `token`, must not be `None`") # noqa: E501 + + self._token = token + + def to_dict(self): + """Returns the model properties as a dict""" + result = {} + + for attr, _ in six.iteritems(self.swagger_types): + value = getattr(self, attr) + if isinstance(value, list): + result[attr] = list(map( + lambda x: x.to_dict() if hasattr(x, "to_dict") else x, + value + )) + elif hasattr(value, "to_dict"): + result[attr] = value.to_dict() + elif isinstance(value, dict): + result[attr] = dict(map( + lambda item: (item[0], item[1].to_dict()) + if hasattr(item[1], "to_dict") else item, + value.items() + )) + else: + result[attr] = value + + return result + + def to_str(self): + """Returns the string representation of the model""" + return pprint.pformat(self.to_dict()) + + def __repr__(self): + """For `print` and `pprint`""" + return self.to_str() + + def __eq__(self, other): + """Returns true if both objects are equal""" + if not isinstance(other, AuthOidcResult): + return False + + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + """Returns true if both objects are not equal""" + return not self == other diff --git a/python/fatcat_client/models/editgroup.py b/python/fatcat_client/models/editgroup.py index 5b1573ed..4c877685 100644 --- a/python/fatcat_client/models/editgroup.py +++ b/python/fatcat_client/models/editgroup.py @@ -60,7 +60,8 @@ class Editgroup(object): if editgroup_id is not None: self.editgroup_id = editgroup_id - self.editor_id = editor_id + if editor_id is not None: + self.editor_id = editor_id if description is not None: self.description = description if extra is not None: @@ -117,8 +118,6 @@ class Editgroup(object): :param editor_id: The editor_id of this Editgroup. # noqa: E501 :type: str """ - if editor_id is None: - raise ValueError("Invalid value for `editor_id`, must not be `None`") # noqa: E501 if editor_id is not None and len(editor_id) > 26: raise ValueError("Invalid value for `editor_id`, length must be less than or equal to `26`") # noqa: E501 if editor_id is not None and len(editor_id) < 26: diff --git a/python/fatcat_client/models/editor.py b/python/fatcat_client/models/editor.py index 2010d454..493f5d81 100644 --- a/python/fatcat_client/models/editor.py +++ b/python/fatcat_client/models/editor.py @@ -32,24 +32,39 @@ class Editor(object): """ swagger_types = { 'editor_id': 'str', - 'username': 'str' + 'username': 'str', + 'is_admin': 'bool', + 'is_bot': 'bool', + 'is_active': 'bool' } attribute_map = { 'editor_id': 'editor_id', - 'username': 'username' + 'username': 'username', + 'is_admin': 'is_admin', + 'is_bot': 'is_bot', + 'is_active': 'is_active' } - def __init__(self, editor_id=None, username=None): # noqa: E501 + def __init__(self, editor_id=None, username=None, is_admin=None, is_bot=None, is_active=None): # noqa: E501 """Editor - a model defined in Swagger""" # noqa: E501 self._editor_id = None self._username = None + self._is_admin = None + self._is_bot = None + self._is_active = None self.discriminator = None if editor_id is not None: self.editor_id = editor_id self.username = username + if is_admin is not None: + self.is_admin = is_admin + if is_bot is not None: + self.is_bot = is_bot + if is_active is not None: + self.is_active = is_active @property def editor_id(self): @@ -103,6 +118,69 @@ class Editor(object): self._username = username + @property + def is_admin(self): + """Gets the is_admin of this Editor. # noqa: E501 + + + :return: The is_admin of this Editor. # noqa: E501 + :rtype: bool + """ + return self._is_admin + + @is_admin.setter + def is_admin(self, is_admin): + """Sets the is_admin of this Editor. + + + :param is_admin: The is_admin of this Editor. # noqa: E501 + :type: bool + """ + + self._is_admin = is_admin + + @property + def is_bot(self): + """Gets the is_bot of this Editor. # noqa: E501 + + + :return: The is_bot of this Editor. # noqa: E501 + :rtype: bool + """ + return self._is_bot + + @is_bot.setter + def is_bot(self, is_bot): + """Sets the is_bot of this Editor. + + + :param is_bot: The is_bot of this Editor. # noqa: E501 + :type: bool + """ + + self._is_bot = is_bot + + @property + def is_active(self): + """Gets the is_active of this Editor. # noqa: E501 + + + :return: The is_active of this Editor. # noqa: E501 + :rtype: bool + """ + return self._is_active + + @is_active.setter + def is_active(self, is_active): + """Sets the is_active of this Editor. + + + :param is_active: The is_active of this Editor. # noqa: E501 + :type: bool + """ + + self._is_active = is_active + def to_dict(self): """Returns the model properties as a dict""" result = {} diff --git a/python/fatcat_export.py b/python/fatcat_export.py index 7d2a6508..cf8bf1c3 100755 --- a/python/fatcat_export.py +++ b/python/fatcat_export.py @@ -13,17 +13,17 @@ import argparse import fatcat_client from fatcat_client.rest import ApiException from fatcat_client import ReleaseEntity -from fatcat_tools import uuid2fcid, entity_from_json, release_to_elasticsearch +from fatcat_tools import uuid2fcid, entity_from_json, entity_to_dict, \ + release_to_elasticsearch, public_api -def run_export_releases(args): - conf = fatcat_client.Configuration() - conf.host = args.host_url - api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) +def run_export_releases(args): + api = args.api for line in args.ident_file: ident = uuid2fcid(line.split()[0]) release = api.get_release(ident=ident, expand="all") - args.json_output.write(json.dumps(release.to_dict()) + "\n") + args.json_output.write( + json.dumps(entity_to_dict(release)) + "\n") def run_transform_releases(args): for line in args.json_input: @@ -35,10 +35,7 @@ def run_transform_releases(args): json.dumps(release_to_elasticsearch(release)) + '\n') def run_export_changelog(args): - conf = fatcat_client.Configuration() - conf.host = args.host_url - api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) - + api = args.api end = args.end if end is None: latest = api.get_changelog(limit=1)[0] @@ -46,7 +43,8 @@ def run_export_changelog(args): for i in range(args.start, end): entry = api.get_changelog_entry(index=i) - args.json_output.write(json.dumps(entry.to_dict()) + "\n") + args.json_output.write( + json.dumps(entity_to_dict(entry)) + "\n") def main(): parser = argparse.ArgumentParser() @@ -92,6 +90,8 @@ def main(): if not args.__dict__.get("func"): print("tell me what to do!") sys.exit(-1) + + args.api = public_api(args.host_url) args.func(args) if __name__ == '__main__': diff --git a/python/fatcat_harvest.py b/python/fatcat_harvest.py index 5f6f471b..e28c9b08 100755 --- a/python/fatcat_harvest.py +++ b/python/fatcat_harvest.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 import sys +import raven import argparse import datetime from fatcat_tools.harvest import HarvestCrossrefWorker, HarvestDataciteWorker,\ HarvestArxivWorker, HarvestPubmedWorker, HarvestDoajArticleWorker,\ HarvestDoajJournalWorker +# Yep, a global. Gets DSN from `SENTRY_DSN` environment variable +sentry_client = raven.Client() + + def run_crossref(args): worker = HarvestCrossrefWorker( kafka_hosts=args.kafka_hosts, diff --git a/python/fatcat_import.py b/python/fatcat_import.py index fe5b24a6..0e176b2c 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 -import sys -import argparse +""" +""" + +import os, sys, argparse +from fatcat_tools import authenticated_api from fatcat_tools.importers import CrossrefImporter, OrcidImporter, \ IssnImporter, MatchedImporter, GrobidMetadataImporter, make_kafka_consumer def run_crossref(args): - fci = CrossrefImporter(args.host_url, args.issn_map_file, - args.extid_map_file, create_containers=(not args.no_create_containers), + fci = CrossrefImporter(args.api, args.issn_map_file, + extid_map_file=args.extid_map_file, + create_containers=(not args.no_create_containers), check_existing=(not args.no_release_updates)) if args.kafka_mode: consumer = make_kafka_consumer( @@ -19,23 +23,23 @@ def run_crossref(args): fci.describe_run() def run_orcid(args): - foi = OrcidImporter(args.host_url) + foi = OrcidImporter(args.api) foi.process_batch(args.json_file, size=args.batch_size) foi.describe_run() def run_issn(args): - fii = IssnImporter(args.host_url) + fii = IssnImporter(args.api) fii.process_csv_batch(args.csv_file, size=args.batch_size) fii.describe_run() def run_matched(args): - fmi = MatchedImporter(args.host_url, + fmi = MatchedImporter(args.api, skip_file_updates=args.no_file_updates) fmi.process_batch(args.json_file, size=args.batch_size) fmi.describe_run() def run_grobid_metadata(args): - fmi = GrobidMetadataImporter(args.host_url) + fmi = GrobidMetadataImporter(args.api) fmi.process_source(args.tsv_file, group_size=args.group_size) fmi.describe_run() @@ -56,7 +60,10 @@ def main(): subparsers = parser.add_subparsers() sub_crossref = subparsers.add_parser('crossref') - sub_crossref.set_defaults(func=run_crossref) + sub_crossref.set_defaults( + func=run_crossref, + auth_var="FATCAT_AUTH_WORKER_CROSSREF", + ) sub_crossref.add_argument('json_file', help="crossref JSON file to import from", default=sys.stdin, type=argparse.FileType('r')) @@ -80,7 +87,10 @@ def main(): help="don't lookup existing DOIs, just insert (only for bootstrap)") sub_orcid = subparsers.add_parser('orcid') - sub_orcid.set_defaults(func=run_orcid) + sub_orcid.set_defaults( + func=run_orcid, + auth_var="FATCAT_AUTH_WORKER_ORCID" + ) sub_orcid.add_argument('json_file', help="orcid JSON file to import from (or stdin)", default=sys.stdin, type=argparse.FileType('r')) @@ -89,7 +99,10 @@ def main(): default=50, type=int) sub_issn = subparsers.add_parser('issn') - sub_issn.set_defaults(func=run_issn) + sub_issn.set_defaults( + func=run_issn, + auth_var="FATCAT_AUTH_WORKER_ISSN", + ) sub_issn.add_argument('csv_file', help="Journal ISSN CSV metadata file to import from (or stdin)", default=sys.stdin, type=argparse.FileType('r')) @@ -98,7 +111,10 @@ def main(): default=50, type=int) sub_matched = subparsers.add_parser('matched') - sub_matched.set_defaults(func=run_matched) + sub_matched.set_defaults( + func=run_matched, + auth_var="FATCAT_AUTH_WORKER_MATCHED", + ) sub_matched.add_argument('json_file', help="JSON file to import from (or stdin)", default=sys.stdin, type=argparse.FileType('r')) @@ -110,7 +126,10 @@ def main(): default=50, type=int) sub_grobid_metadata = subparsers.add_parser('grobid-metadata') - sub_grobid_metadata.set_defaults(func=run_grobid_metadata) + sub_grobid_metadata.set_defaults( + func=run_grobid_metadata, + auth_var="FATCAT_AUTH_WORKER_GROBID_METADATA", + ) sub_grobid_metadata.add_argument('tsv_file', help="TSV file to import from (or stdin)", default=sys.stdin, type=argparse.FileType('r')) @@ -122,6 +141,10 @@ def main(): if not args.__dict__.get("func"): print("tell me what to do!") sys.exit(-1) + + args.api = authenticated_api( + args.host_url, + token=os.environ.get(args.auth_var)) args.func(args) if __name__ == '__main__': diff --git a/python/fatcat_tools/__init__.py b/python/fatcat_tools/__init__.py index 0bb42ab5..e2b1e3a2 100644 --- a/python/fatcat_tools/__init__.py +++ b/python/fatcat_tools/__init__.py @@ -1,3 +1,4 @@ +from .api_auth import authenticated_api, public_api from .fcid import fcid2uuid, uuid2fcid -from .transforms import entity_to_json, entity_from_json, release_to_elasticsearch +from .transforms import entity_to_dict, entity_from_json, release_to_elasticsearch diff --git a/python/fatcat_tools/api_auth.py b/python/fatcat_tools/api_auth.py new file mode 100644 index 00000000..c49051f6 --- /dev/null +++ b/python/fatcat_tools/api_auth.py @@ -0,0 +1,40 @@ + +import os, sys +import fatcat_client +from fatcat_client.rest import ApiException + + +def public_api(host_uri): + """ + Note: unlike the authenticated variant, this helper might get called even + if the API isn't going to be used, so it's important that it doesn't try to + actually connect to the API host or something. + """ + conf = fatcat_client.Configuration() + conf.host = host_uri + return fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) + +def authenticated_api(host_uri, token=None): + """ + Note: if this helper is called, it's implied that an actual API connection + is needed, so it does try to connect and verify credentials. + """ + + conf = fatcat_client.Configuration() + conf.host = host_uri + if not token: + token = os.environ['FATCAT_API_AUTH_TOKEN'] + if not token: + sys.stderr.write( + 'This client requires a fatcat API token (eg, in env var FATCAT_API_AUTH_TOKEN)\n') + sys.exit(-1) + + conf.api_key["Authorization"] = token + conf.api_key_prefix["Authorization"] = "Bearer" + api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) + + # verify up front that auth is working + api.auth_check() + + return api + diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index e31cabf8..e39ec6c9 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -4,6 +4,7 @@ import sys import csv import json import itertools +import subprocess from collections import Counter import pykafka @@ -37,19 +38,33 @@ class FatcatImporter: Base class for fatcat importers """ - def __init__(self, host_url, issn_map_file=None): - conf = fatcat_client.Configuration() - conf.host = host_url - self.api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) + def __init__(self, api, **kwargs): + + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['git_rev'] = eg_extra.get('git_rev', + subprocess.check_output(["git", "describe", "--always"]).strip()).decode('utf-8') + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.FatcatImporter') + + self.api = api + self._editgroup_description = kwargs.get('editgroup_description') + self._editgroup_extra = kwargs.get('editgroup_extra') + issn_map_file = kwargs.get('issn_map_file') + self._issnl_id_map = dict() self._orcid_id_map = dict() self._doi_id_map = dict() - self._issn_issnl_map = None - self._orcid_regex = re.compile("^\\d{4}-\\d{4}-\\d{4}-\\d{3}[\\dX]$") if issn_map_file: self.read_issn_map_file(issn_map_file) + self._orcid_regex = re.compile("^\\d{4}-\\d{4}-\\d{4}-\\d{3}[\\dX]$") self.counts = Counter({'insert': 0, 'update': 0, 'processed_lines': 0}) + def _editgroup(self): + eg = fatcat_client.Editgroup( + description=self._editgroup_description, + extra=self._editgroup_extra, + ) + return self.api.create_editgroup(eg) + def describe_run(self): print("Processed {} lines, inserted {}, updated {}.".format( self.counts['processed_lines'], self.counts['insert'], self.counts['update'])) @@ -64,15 +79,13 @@ class FatcatImporter: def process_source(self, source, group_size=100): """Creates and auto-accepts editgroup every group_size rows""" - eg = self.api.create_editgroup( - fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) + eg = self._editgroup() i = 0 for i, row in enumerate(source): self.create_row(row, editgroup_id=eg.editgroup_id) if i > 0 and (i % group_size) == 0: self.api.accept_editgroup(eg.editgroup_id) - eg = self.api.create_editgroup( - fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) + eg = self._editgroup() self.counts['processed_lines'] += 1 if i == 0 or (i % group_size) != 0: self.api.accept_editgroup(eg.editgroup_id) @@ -83,8 +96,7 @@ class FatcatImporter: if decode_kafka: rows = [msg.value.decode('utf-8') for msg in rows] self.counts['processed_lines'] += len(rows) - eg = self.api.create_editgroup( - fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) + eg = self._editgroup() self.create_batch(rows, editgroup_id=eg.editgroup_id) def process_csv_source(self, source, group_size=100, delimiter=','): diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py index d4d0de68..ed60a78c 100644 --- a/python/fatcat_tools/importers/crossref.py +++ b/python/fatcat_tools/importers/crossref.py @@ -4,6 +4,7 @@ import json import sqlite3 import datetime import itertools +import subprocess import fatcat_client from .common import FatcatImporter @@ -40,8 +41,19 @@ class CrossrefImporter(FatcatImporter): See https://github.com/CrossRef/rest-api-doc for JSON schema notes """ - def __init__(self, host_url, issn_map_file, extid_map_file=None, create_containers=True, check_existing=True): - super().__init__(host_url, issn_map_file) + def __init__(self, api, issn_map_file, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of Crossref DOI metadata, harvested from REST API") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.CrossrefImporter') + super().__init__(api, + issn_map_file=issn_map_file, + editgroup_description=eg_desc, + editgroup_extra=eg_extra) + extid_map_file = kwargs.get('extid_map_file') + create_containers = kwargs.get('create_containers') + check_existing = kwargs.get('check_existing') self.extid_map_db = None if extid_map_file: db_uri = "file:{}?mode=ro".format(extid_map_file) @@ -313,8 +325,7 @@ class CrossrefImporter(FatcatImporter): if entities is not None: (re, ce) = entities if ce is not None: - ce_eg = self.api.create_editgroup( - fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) + ce_eg = self.api.create_editgroup(fatcat_client.Editgroup()) container = self.api.create_container(ce, editgroup_id=ce_eg.editgroup_id) self.api.accept_editgroup(ce_eg.editgroup_id) re.container_id = container.ident diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py index 2cb97b01..5e61a154 100644 --- a/python/fatcat_tools/importers/grobid_metadata.py +++ b/python/fatcat_tools/importers/grobid_metadata.py @@ -12,9 +12,16 @@ MAX_ABSTRACT_BYTES=4096 class GrobidMetadataImporter(FatcatImporter): - def __init__(self, host_url, default_link_rel="web"): - super().__init__(host_url) - self.default_link_rel = default_link_rel + def __init__(self, api, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Import of release and file metadata, as extracted from PDFs by GROBID.") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.GrobidMetadataImporter') + super().__init__(api, + editgroup_description=eg_desc, + editgroup_extra=eg_extra) + self.default_link_rel = kwargs.get("default_link_rel", "web") def parse_grobid_json(self, obj): diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py index 9b9ca63f..02a1eea0 100644 --- a/python/fatcat_tools/importers/issn.py +++ b/python/fatcat_tools/importers/issn.py @@ -35,6 +35,16 @@ class IssnImporter(FatcatImporter): ISSN-L,in_doaj,in_road,in_norwegian,in_crossref,title,publisher,url,lang,ISSN-print,ISSN-electronic,doi_count,has_doi,is_oa,is_kept,publisher_size,url_live,url_live_status,url_live_final_status,url_live_final_url,url_live_status_simple,url_live_final_status_simple,url_domain,gwb_pdf_count """ + def __init__(self, api, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of container-level metadata, by ISSN. Metadata from Internet Archive munging.") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.IssnImporter') + super().__init__(api, + editgroup_description=eg_desc, + editgroup_extra=eg_extra) + def parse_issn_row(self, row): """ row is a python dict (parsed from CSV). diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py index 5dbda27c..0b77bcf0 100644 --- a/python/fatcat_tools/importers/matched.py +++ b/python/fatcat_tools/importers/matched.py @@ -37,12 +37,18 @@ class MatchedImporter(FatcatImporter): - core_id, wikidata_id, pmcid, pmid: not as lists """ - def __init__(self, host_url, skip_file_updates=False, default_mime=None, - default_link_rel="web"): - super().__init__(host_url) - self.default_mime = default_mime - self.default_link_rel = default_link_rel - self.skip_file_updates = skip_file_updates + def __init__(self, api, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Import of large-scale file-to-release match results. Source of metadata varies.") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.MatchedImporter') + super().__init__(api, + editgroup_description=eg_desc, + editgroup_extra=eg_extra) + self.default_link_rel = kwargs.get("default_link_rel", "web") + self.default_mime = kwargs.get("default_mime", None) + self.skip_file_updates = kwargs.get("skip_file_updates", False) def make_url(self, raw): rel = self.default_link_rel diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index fc4562d0..0aa4ab00 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -22,6 +22,16 @@ def value_or_none(e): class OrcidImporter(FatcatImporter): + def __init__(self, api, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of ORCID metadata, from official bulk releases.") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.OrcidImporter') + super().__init__(api, + editgroup_description=eg_desc, + editgroup_extra=eg_extra) + def parse_orcid_dict(self, obj): """ obj is a python dict (parsed from json). diff --git a/python/fatcat_tools/transforms.py b/python/fatcat_tools/transforms.py index 843c00a5..0f957f9a 100644 --- a/python/fatcat_tools/transforms.py +++ b/python/fatcat_tools/transforms.py @@ -2,7 +2,7 @@ import collections from fatcat_client import ReleaseEntity, ApiClient -def entity_to_json(entity): +def entity_to_dict(entity): """ Hack to take advantage of the code-generated serialization code """ diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index b6d99d06..8690a791 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -12,11 +12,11 @@ class ChangelogWorker(FatcatWorker): found, fetch them and push (as JSON) into a Kafka topic. """ - def __init__(self, api_host_url, kafka_hosts, produce_topic, poll_interval=10.0, offset=None): + def __init__(self, api, kafka_hosts, produce_topic, poll_interval=10.0, offset=None): # TODO: should be offset=0 super().__init__(kafka_hosts=kafka_hosts, produce_topic=produce_topic, - api_host_url=api_host_url) + api=api) self.poll_interval = poll_interval self.offset = offset # the fatcat changelog offset, not the kafka offset @@ -61,10 +61,10 @@ class EntityUpdatesWorker(FatcatWorker): For now, only release updates are published. """ - def __init__(self, api_host_url, kafka_hosts, consume_topic, release_topic): + def __init__(self, api, kafka_hosts, consume_topic, release_topic): super().__init__(kafka_hosts=kafka_hosts, consume_topic=consume_topic, - api_host_url=api_host_url) + api=api) self.release_topic = release_topic self.consumer_group = "entity-updates" diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py index e400e815..b84341c7 100644 --- a/python/fatcat_tools/workers/worker_common.py +++ b/python/fatcat_tools/workers/worker_common.py @@ -45,11 +45,9 @@ class FatcatWorker: Common code for for Kafka producers and consumers. """ - def __init__(self, kafka_hosts, produce_topic=None, consume_topic=None, api_host_url=None): - if api_host_url: - conf = fatcat_client.Configuration() - conf.host = api_host_url - self.api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) + def __init__(self, kafka_hosts, produce_topic=None, consume_topic=None, api=None): + if api: + self.api = api self.kafka = KafkaClient(hosts=kafka_hosts, broker_version="1.0.0") self.produce_topic = produce_topic self.consume_topic = consume_topic diff --git a/python/fatcat_web/__init__.py b/python/fatcat_web/__init__.py index 3c790e7a..cd7af195 100644 --- a/python/fatcat_web/__init__.py +++ b/python/fatcat_web/__init__.py @@ -2,21 +2,47 @@ from flask import Flask from flask_uuid import FlaskUUID from flask_debugtoolbar import DebugToolbarExtension +from flask_login import LoginManager +from authlib.flask.client import OAuth +from loginpass import create_flask_blueprint, Gitlab from raven.contrib.flask import Sentry -from web_config import Config import fatcat_client +from fatcat_web.web_config import Config + toolbar = DebugToolbarExtension() app = Flask(__name__) app.config.from_object(Config) toolbar = DebugToolbarExtension(app) FlaskUUID(app) +login_manager = LoginManager() +login_manager.init_app(app) +login_manager.login_view = "/auth/login" +oauth = OAuth(app) + # Grabs sentry config from SENTRY_DSN environment variable sentry = Sentry(app) conf = fatcat_client.Configuration() -conf.host = "http://localhost:9411/v0" +conf.host = Config.FATCAT_API_HOST api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) -from fatcat_web import routes +def auth_api(token): + conf = fatcat_client.Configuration() + conf.api_key["Authorization"] = token + conf.api_key_prefix["Authorization"] = "Bearer" + conf.host = Config.FATCAT_API_HOST + return fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) + +if Config.FATCAT_API_AUTH_TOKEN: + print("Found and using privileged token (eg, for account signup)") + priv_api = auth_api(Config.FATCAT_API_AUTH_TOKEN) +else: + print("No privileged token found") + priv_api = None + +from fatcat_web import routes, auth + +gitlab_bp = create_flask_blueprint(Gitlab, oauth, auth.handle_oauth) +app.register_blueprint(gitlab_bp, url_prefix='/auth/gitlab') diff --git a/python/fatcat_web/auth.py b/python/fatcat_web/auth.py new file mode 100644 index 00000000..8035cbe5 --- /dev/null +++ b/python/fatcat_web/auth.py @@ -0,0 +1,141 @@ + +from collections import namedtuple +import requests +import pymacaroons +from flask import Flask, render_template, send_from_directory, request, \ + url_for, abort, g, redirect, jsonify, session, flash +from fatcat_web import login_manager, api, priv_api, Config +from flask_login import logout_user, login_user, UserMixin +import fatcat_client + +def handle_logout(): + logout_user() + for k in ('editor', 'api_token'): + if k in session: + session.pop(k) + session.clear() + +def handle_token_login(token): + try: + m = pymacaroons.Macaroon.deserialize(token) + except pymacaroons.exceptions.MacaroonDeserializationException: + # TODO: what kind of Exceptions? + return abort(400) + # extract editor_id + editor_id = None + for caveat in m.first_party_caveats(): + caveat = caveat.caveat_id + if caveat.startswith(b"editor_id = "): + editor_id = caveat[12:].decode('utf-8') + if not editor_id: + abort(400) + # fetch editor info + editor = api.get_editor(editor_id) + session.permanent = True + session['api_token'] = token + session['editor'] = editor.to_dict() + login_user(load_user(editor.editor_id)) + return redirect("/auth/account") + +# This will need to login/signup via fatcatd API, then set token in session +def handle_oauth(remote, token, user_info): + if user_info: + # fetch api login/signup using user_info + # ISS is basically the API url (though more formal in OIDC) + # SUB is the stable internal identifier for the user (not usually the username itself) + # TODO: should have the real sub here + # TODO: would be nicer to pass preferred_username for account creation + iss = remote.OAUTH_CONFIG['api_base_url'] + + # we reuse 'preferred_username' for account name auto-creation (but + # don't store it otherwise in the backend, at least currently). But i'm + # not sure all loginpass backends will set it + if user_info.get('preferred_username'): + preferred_username = user_info['preferred_username'] + else: + preferred_username = user_info['sub'] + + params = fatcat_client.AuthOidc(remote.name, user_info['sub'], iss, user_info['preferred_username']) + # this call requires admin privs + (resp, http_status, http_headers) = priv_api.auth_oidc_with_http_info(params) + editor = resp.editor + api_token = resp.token + + if http_status == 201: + flash("Welcome to Fatcat! An account has been created for you with a temporary username; you may wish to change it under account settings") + flash("You must use the same mechanism ({}) to login in the future".format(remote.name)) + else: + flash("Welcome back!") + + # write token and username to session + session.permanent = True + session['api_token'] = api_token + session['editor'] = editor.to_dict() + + # call login_user(load_user(editor_id)) + login_user(load_user(editor.editor_id)) + return redirect("/auth/account") + + # XXX: what should this actually be? + raise Exception("didn't receive OAuth user_info") + +def handle_ia_xauth(email, password): + resp = requests.post(Config.IA_XAUTH_URI, + params={'op': 'authenticate'}, + json={ + 'version': '1', + 'email': email, + 'password': password, + 'access': Config.IA_XAUTH_CLIENT_ID, + 'secret': Config.IA_XAUTH_CLIENT_SECRET, + }) + if resp.status_code == 401 or (not resp.json().get('success')): + flash("Internet Archive email/password didn't match: {}".format(resp.json()['values']['reason'])) + return render_template('auth_ia_login.html', email=email), resp.status_code + elif resp.status_code != 200: + flash("Internet Archive login failed (internal error?)") + # TODO: log.warn + print("IA XAuth fail: {}".format(resp.content)) + return render_template('auth_ia_login.html', email=email), resp.status_code + + # Successful login; now fetch info... + resp = requests.post(Config.IA_XAUTH_URI, + params={'op': 'info'}, + json={ + 'version': '1', + 'email': email, + 'access': Config.IA_XAUTH_CLIENT_ID, + 'secret': Config.IA_XAUTH_CLIENT_SECRET, + }) + if resp.status_code != 200: + flash("Internet Archive login failed (internal error?)") + # TODO: log.warn + print("IA XAuth fail: {}".format(resp.content)) + return render_template('auth_ia_login.html', email=email), resp.status_code + ia_info = resp.json()['values'] + + # and pass off "as if" we did OAuth successfully + FakeOAuthRemote = namedtuple('FakeOAuthRemote', ['name', 'OAUTH_CONFIG']) + remote = FakeOAuthRemote(name='archive', OAUTH_CONFIG={'api_base_url': Config.IA_XAUTH_URI}) + oauth_info = { + 'preferred_username': ia_info['screenname'], + 'iss': Config.IA_XAUTH_URI, + 'sub': ia_info['itemname'], + } + return handle_oauth(remote, None, oauth_info) + +@login_manager.user_loader +def load_user(editor_id): + # looks for extra info in session, and updates the user object with that. + # If session isn't loaded/valid, should return None + if (not session.get('editor')) or (not session.get('api_token')): + return None + editor = session['editor'] + token = session['api_token'] + user = UserMixin() + user.id = editor_id + user.editor_id = editor_id + user.username = editor['username'] + user.token = token + return user + diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 998697bc..789d7bed 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -2,8 +2,10 @@ import os import json from flask import Flask, render_template, send_from_directory, request, \ - url_for, abort, g, redirect, jsonify, session -from fatcat_web import app, api + url_for, abort, g, redirect, jsonify, session, flash +from flask_login import login_required +from fatcat_web import app, api, auth_api +from fatcat_web.auth import handle_token_login, handle_logout, load_user, handle_ia_xauth from fatcat_client.rest import ApiException from fatcat_web.search import do_search @@ -295,12 +297,6 @@ def work_view(ident): return render_template('deleted_entity.html', entity=entity) return render_template('work_view.html', work=entity, releases=releases) -@app.route('/editgroup/current', methods=['GET']) -def editgroup_current(): - raise NotImplementedError - #eg = api.get_or_create_editgroup() - #return redirect('/editgroup/{}'.format(eg.id)) - @app.route('/editgroup/<ident>', methods=['GET']) def editgroup_view(ident): try: @@ -327,6 +323,17 @@ def editor_changelog(ident): return render_template('editor_changelog.html', editor=editor, changelog_entries=changelog_entries) +@app.route('/editor/<ident>/wip', methods=['GET']) +def editor_wip(ident): + raise NotImplementedError + try: + editor = api.get_editor(ident) + entries = api.get_editor_wip(ident) + except ApiException as ae: + abort(ae.status) + return render_template('editor_changelog.html', editor=editor, + entries=entries) + @app.route('/changelog', methods=['GET']) def changelog_view(): try: @@ -367,6 +374,61 @@ def search(): return render_template('release_search.html', query=query, fulltext_only=fulltext_only) +### Auth #################################################################### + +@app.route('/auth/login') +def login(): + # show the user a list of login options + return render_template('auth_login.html') + +@app.route('/auth/ia/login', methods=['GET', 'POST']) +def ia_xauth_login(): + if 'email' in request.form: + # if a login attempt... + return handle_ia_xauth(request.form.get('email'), request.form.get('password')) + # else show form + return render_template('auth_ia_login.html') + +@app.route('/auth/token_login', methods=['GET', 'POST']) +def token_login(): + # show the user a list of login options + if 'token' in request.args: + return handle_token_login(request.args.get('token')) + if 'token' in request.form: + return handle_token_login(request.form.get('token')) + return render_template('auth_token_login.html') + +@app.route('/auth/change_username', methods=['POST']) +@login_required +def change_username(): + # show the user a list of login options + if not 'username' in request.form: + abort(400) + # on behalf of user... + user_api = auth_api(session['api_token']) + editor = user_api.get_editor(session['editor']['editor_id']) + editor.username = request.form['username'] + editor = user_api.update_editor(editor.editor_id, editor) + # update our session + session['editor'] = editor.to_dict() + load_user(editor.editor_id) + flash("Username updated successfully") + return redirect('/auth/account') + +@app.route('/auth/logout') +def logout(): + handle_logout() + return render_template('auth_logout.html') + +@app.route('/auth/account') +@login_required +def auth_account(): + editor = api.get_editor(session['editor']['editor_id']) + session['editor'] = editor.to_dict() + load_user(editor.editor_id) + return render_template('auth_account.html') + + ### Static Routes ########################################################### @app.errorhandler(404) diff --git a/python/fatcat_web/templates/auth_account.html b/python/fatcat_web/templates/auth_account.html new file mode 100644 index 00000000..57155722 --- /dev/null +++ b/python/fatcat_web/templates/auth_account.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} +{% block body %} + +<h1>Your Account</h1> + +<p><b>Username:</b> <code>{{ current_user.username }}</code> +<p><b>Editor Id:</b> <code><a href="/editor/{{ current_user.editor_id }}">{{ current_user.editor_id }}</a></code> + +<div> +<p>Change username: +<form class="" role="change_username" action="/auth/change_username" method="post"> + <div class="ui form"> + <div class="ui action input medium fluid"> + <input type="text" name="username" value="{{ current_user.username }}" aria-label="account username"> + <button class="ui button">Update</button> + </div> + </div> +</form> +</div> + +<p>In the future, you might be able to... +<ul> + <li>Create a bot user + <li>Generate an API token +</ul> + +{% endblock %} diff --git a/python/fatcat_web/templates/auth_ia_login.html b/python/fatcat_web/templates/auth_ia_login.html new file mode 100644 index 00000000..ebf08021 --- /dev/null +++ b/python/fatcat_web/templates/auth_ia_login.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} +{% block body %} +<h1>Login with Internet Archive account</h1> + +<p>Warning: still experimental! + +<br> +<br> +<br> + +{% if current_user.is_authenticated %} + <div class="ui negative message"> + <div class="header">You are already logged in!</div> + <p>You should logout first. Re-authenticating would be undefined behavior. + </div> +{% else %} + <form class="" role="login" action="/auth/ia/login" method="post"> + <div class="ui form"> + <div class="ui input huge fluid"> + <input type="email" placeholder="user@domain.tdl..." name="email" {% if email %}value="{{ email }}"{% endif %} aria-label="email for login"> + </div> + <div class="ui action input huge fluid"> + <input type="password" placeholder="password" name="password" aria-label="internet archive password"> + <button class="ui button">Login</button> + </div> + </div> + </div> + </form> +{% endif %} + +{% endblock %} diff --git a/python/fatcat_web/templates/auth_login.html b/python/fatcat_web/templates/auth_login.html new file mode 100644 index 00000000..9ccae816 --- /dev/null +++ b/python/fatcat_web/templates/auth_login.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% block body %} +<h1>Login</h1> + +<p>via OAuth / OpenID Connect: +<ul> + <li><a href="/auth/gitlab/login">gitlab.com</a> + <li><strike><a href="/auth/google/login">google.com</a></strike> + <li><strike><a href="/auth/orcid/login">orcid.org</a></strike> +</ul> + +<p>Other options... +<ul> + <li><a href="/auth/token_login">Using auth token</a> (admin/operator) + <li><a href="/auth/ia/login">With Internet Archive account</a> (experimental) +</ul> + +{% endblock %} diff --git a/python/fatcat_web/templates/auth_logout.html b/python/fatcat_web/templates/auth_logout.html new file mode 100644 index 00000000..819d42fe --- /dev/null +++ b/python/fatcat_web/templates/auth_logout.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} +{% block body %} +<h1>Logout</h1> + +<p>If you are seeing this page, you are now logged out. + +<p>Use the links above to return to the home page or log back in. +{% endblock %} diff --git a/python/fatcat_web/templates/auth_token_login.html b/python/fatcat_web/templates/auth_token_login.html new file mode 100644 index 00000000..4c28f938 --- /dev/null +++ b/python/fatcat_web/templates/auth_token_login.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} +{% block body %} +<h1>Login with Token</h1> + +<p>This page is intended for operators and contingencies, not for general use. It +allows editors (users) to use an existing token (macaroon) for authentication; +a new web interface session and cookie are constructed using the token. + +<br> +<br> +<br> + +{% if current_user.is_authenticated %} + <div class="ui negative message"> + <div class="header">You are already logged in!</div> + <p>You should logout first. Re-authenticating would be undefined behavior. + </div> +{% else %} + <form class="" role="login" action="/auth/token_login" method="post"> + <div class="ui form"> + <div class="ui action input huge fluid"> + <input type="password" placeholder="Your Fatcat API Auth Token..." name="token" value="{% if token %}{{ token }}{% endif %}" aria-label="login using token"> + <button class="ui button">Login</button> + </div> + </div> + </form> +{% endif %} + +{% endblock %} diff --git a/python/fatcat_web/templates/base.html b/python/fatcat_web/templates/base.html index 4b3b7e0b..cce841e5 100644 --- a/python/fatcat_web/templates/base.html +++ b/python/fatcat_web/templates/base.html @@ -29,23 +29,41 @@ </div> </form> </div> +{% if current_user.is_authenticated %} <div class="ui simple dropdown item"> - demo-user <i class="dropdown icon"></i> + {{ current_user.username }} <i class="dropdown icon"></i> <div class="menu"> - <a class="item" href="/editgroup/current"><i class="edit icon"></i>Edits in Progress</a> - <a class="item" href="/editor/aaaaaaaaaaaabkvkaaaaaaaaae/changelog"><i class="history icon"></i>History</a> + <a class="item" href="#"><i class="edit icon"></i>Edits in Progress</a> + <a class="item" href="/editor/{{ current_user.editor_id }}/changelog"><i class="history icon"></i>History</a> <div class="divider"></div> - <a class="item" href="/editor/aaaaaaaaaaaabkvkaaaaaaaaae"><i class="user icon"></i>Account</a> - <a class="item" href="/logout"><i class="sign out icon"></i>Logout</a> + <a class="item" href="/auth/account"><i class="user icon"></i>Account</a> + <a class="item" href="/auth/logout"><i class="sign out icon"></i>Logout</a> </div> </div> - +{% else %} + <div class="ui simple item"> + <a href="/auth/login">Login/Signup</a> + </div> +{% endif %} </div> </div> </header> <!-- 4em top margin is "enough" --> <main class="ui main container" style="margin-top: 6em; margin-bottom: 2em;"> +{% with messages = get_flashed_messages() %} + {% if messages %} + <div class="ui message"> + {# Needs more javascript: <i class="close icon"></i> #} + <div class="header">Now Hear This...</div> + <ul class="list"> + {% for message in messages %} + <li>{{ message }} + {% endfor %} + </ul> + </div> + {% endif %} +{% endwith %} {% block fullbody %} <div class="ui container text"> {% block body %}Nothing to see here.{% endblock %} @@ -62,7 +80,7 @@ <a class="item" href="https://guide.{{ config.FATCAT_DOMAIN }}/sources.html">Sources</a> <a class="item" href="{% if config.FATCAT_DOMAIN == "fatcat.wiki" %}https://stats.uptimerobot.com/GM9YNSrB0{% elif config.FATCAT_DOMAIN =="qa.fatcat.wiki" %}https://stats.uptimerobot.com/WQ8wAUREA{% else %}#{% endif %}">Status</a> <a class="item" href="https://guide.{{ config.FATCAT_DOMAIN }}/bulk_exports.html">Bulk Exports</a> - <a class="item" href="https://github.com/internetarchive/fatcat/">Source Code (<code>{{ config.GIT_REVISION.decode() }}</code>)</a> + <a class="item" href="https://github.com/internetarchive/fatcat/">Source Code (<code>{{ config.GIT_REVISION }}</code>)</a> </div> </div> </footer> diff --git a/python/fatcat_web/templates/editor_changelog.html b/python/fatcat_web/templates/editor_changelog.html index 79127312..785c19bd 100644 --- a/python/fatcat_web/templates/editor_changelog.html +++ b/python/fatcat_web/templates/editor_changelog.html @@ -3,8 +3,8 @@ <h1 class="ui header">Editor Changelog: {{ editor.username }} <div class="sub header"> - <a href="/editor/{{editor.id}}"> - <code>editor {{ editor.id }}</code> + <a href="/editor/{{editor.editor_id}}"> + <code>editor {{ editor.editor_id }}</code> </a> </div> </h1> diff --git a/python/fatcat_web/templates/editor_view.html b/python/fatcat_web/templates/editor_view.html index c9b61f5d..eef4f040 100644 --- a/python/fatcat_web/templates/editor_view.html +++ b/python/fatcat_web/templates/editor_view.html @@ -3,10 +3,10 @@ <h1 class="ui header">{{ editor.username }} <div class="sub header"> - <code>editor {{ editor.id }}</code> + <code>editor {{ editor.editor_id }}</code> </div> </h1> -<p><b><a href="/editor/{{ editor.id }}/changelog">View editor's changelog</a></b> +<p><b><a href="/editor/{{ editor.editor_id }}/changelog">View editor's changelog</a></b> {% endblock %} diff --git a/python/web_config.py b/python/fatcat_web/web_config.py index 9df9c205..cbe519b0 100644 --- a/python/web_config.py +++ b/python/fatcat_web/web_config.py @@ -16,13 +16,34 @@ import subprocess basedir = os.path.abspath(os.path.dirname(__file__)) class Config(object): - GIT_REVISION = subprocess.check_output(["git", "describe", "--always"]).strip() + GIT_REVISION = subprocess.check_output(["git", "describe", "--always"]).strip().decode('utf-8') + # This is, effectively, the QA/PROD flag FATCAT_DOMAIN = os.environ.get("FATCAT_DOMAIN", default="qa.fatcat.wiki") + FATCAT_API_AUTH_TOKEN = os.environ.get("FATCAT_API_AUTH_TOKEN", default=None) + FATCAT_API_HOST = os.environ.get("FATCAT_API_HOST", default="https://{}/v0".format(FATCAT_DOMAIN)) + # can set this to https://search.fatcat.wiki for some experimentation ELASTICSEARCH_BACKEND = os.environ.get("ELASTICSEARCH_BACKEND", default="http://localhost:9200") ELASTICSEARCH_INDEX = os.environ.get("ELASTICSEARCH_INDEX", default="fatcat") + # for flask things, like session cookies + FLASK_SECRET_KEY = os.environ.get("FLASK_SECRET_KEY", default=None) + SECRET_KEY = FLASK_SECRET_KEY + + GITLAB_CLIENT_ID = os.environ.get("GITLAB_CLIENT_ID", default=None) + GITLAB_CLIENT_SECRET = os.environ.get("GITLAB_CLIENT_SECRET", default=None) + + IA_XAUTH_URI = "https://archive.org/services/xauthn/" + IA_XAUTH_CLIENT_ID = os.environ.get("IA_XAUTH_CLIENT_ID", default=None) + IA_XAUTH_CLIENT_SECRET = os.environ.get("IA_XAUTH_CLIENT_SECRET", default=None) + + # protect cookies (which include API tokens) + SESSION_COOKIE_HTTPONLY = True + SESSION_COOKIE_SECURE = True + SESSION_COOKIE_SAMESITE = 'Lax' + PERMANENT_SESSION_LIFETIME = 2678400 # 31 days, in seconds + try: GIT_RELEASE = raven.fetch_git_sha('..') except Exception as e: @@ -38,7 +59,6 @@ class Config(object): }, } - # "Event more verbose" debug options. SECRET_KEY is bogus. + # "Even more verbose" debug options #SQLALCHEMY_ECHO = True - #SECRET_KEY = "kuhy0284hflskjhg01284" #DEBUG = True diff --git a/python/fatcat_worker.py b/python/fatcat_worker.py index e0ac48d8..0207fb19 100755 --- a/python/fatcat_worker.py +++ b/python/fatcat_worker.py @@ -1,28 +1,33 @@ #!/usr/bin/env python3 import sys +import raven import argparse import datetime +from fatcat_tools import public_api from fatcat_tools.workers import ChangelogWorker, EntityUpdatesWorker, ElasticsearchReleaseWorker +# Yep, a global. Gets DSN from `SENTRY_DSN` environment variable +sentry_client = raven.Client() + def run_changelog(args): topic = "fatcat-{}.changelog".format(args.env) - worker = ChangelogWorker(args.api_host_url, args.kafka_hosts, topic, - args.poll_interval) + worker = ChangelogWorker(args.api, args.kafka_hosts, topic, + poll_interval=args.poll_interval) worker.run() def run_entity_updates(args): changelog_topic = "fatcat-{}.changelog".format(args.env) release_topic = "fatcat-{}.release-updates".format(args.env) - worker = EntityUpdatesWorker(args.api_host_url, args.kafka_hosts, - changelog_topic, release_topic) + worker = EntityUpdatesWorker(args.api, args.kafka_hosts, changelog_topic, + release_topic=release_topic) worker.run() def run_elasticsearch_release(args): consume_topic = "fatcat-{}.release-updates".format(args.env) - worker = ElasticsearchReleaseWorker(args.kafka_hosts, - consume_topic, elasticsearch_backend=args.elasticsearch_backend, + worker = ElasticsearchReleaseWorker(args.kafka_hosts, consume_topic, + elasticsearch_backend=args.elasticsearch_backend, elasticsearch_index=args.elasticsearch_index) worker.run() @@ -64,6 +69,8 @@ def main(): if not args.__dict__.get("func"): print("tell me what to do!") sys.exit(-1) + + args.api = public_api(args.api_host_url) args.func(args) if __name__ == '__main__': diff --git a/python/shell.py b/python/shell.py index 78d32deb..ad92f4ae 100644 --- a/python/shell.py +++ b/python/shell.py @@ -28,14 +28,23 @@ if __name__ == '__main__': admin_id = "aaaaaaaaaaaabkvkaaaaaaaaae" + #fatcat_client.configuration.api_key['Authorization'] = 'YOUR_API_KEY' + #fatcat_client.configuration.api_key_prefix['Authorization'] = 'Bearer' local_conf = fatcat_client.Configuration() + local_conf.api_key["Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw" + local_conf.api_key_prefix["Authorization"] = "Bearer" local_conf.host = 'http://localhost:9411/v0' + local_conf.debug = True local_api = fatcat_client.DefaultApi(fatcat_client.ApiClient(local_conf)) - prod_conf = fatcat_client.Configuration() - prod_conf.host = 'https://api.fatcat.wiki/v0' - prod_api = fatcat_client.DefaultApi(fatcat_client.ApiClient(prod_conf)) + #prod_conf = fatcat_client.Configuration() + #local_conf.api_key["Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw" + #local_conf.api_key_prefix["Authorization"] = "Bearer" + #prod_conf.host = 'https://api.fatcat.wiki/v0' + #prod_api = fatcat_client.DefaultApi(fatcat_client.ApiClient(prod_conf)) qa_conf = fatcat_client.Configuration() + local_conf.api_key["Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw" + local_conf.api_key_prefix["Authorization"] = "Bearer" qa_conf.host = 'https://api.qa.fatcat.wiki/v0' qa_api = fatcat_client.DefaultApi(fatcat_client.ApiClient(qa_conf)) diff --git a/python/tests/cli.sh b/python/tests/cli.sh new file mode 100755 index 00000000..eba6d3a7 --- /dev/null +++ b/python/tests/cli.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail +set -x + +# This is a helper script to at least partially exersize the fatcat_*.py +# scripts. It expects to be run from the top-level directory, inside a 'pipenv +# shell' or 'pipenv run' invocation. + +./fatcat_export.py changelog --start 1 --end 10 - > /dev/null + +# no easy way to run this without harvest a whole day (at the moment) +./fatcat_harvest.py crossref -h + +./fatcat_import.py crossref tests/files/crossref-works.2018-01-21.badsample.json tests/files/ISSN-to-ISSN-L.snip.txt +./fatcat_import.py orcid tests/files/0000-0001-8254-7103.json +./fatcat_import.py issn tests/files/journal_extra_metadata.snip.csv +./fatcat_import.py matched tests/files/matched_sample.json +./fatcat_import.py matched tests/files/example_matched.json +./fatcat_import.py grobid-metadata tests/files/example_grobid_metadata_lines.tsv + +./fatcat_webface.py -h +./fatcat_worker.py -h + +set +x +echo "Done running CLI examples (SUCCESS)" diff --git a/python/tests/codegen_tests/test_auth_oidc.py b/python/tests/codegen_tests/test_auth_oidc.py new file mode 100644 index 00000000..f799da55 --- /dev/null +++ b/python/tests/codegen_tests/test_auth_oidc.py @@ -0,0 +1,40 @@ +# coding: utf-8 + +""" + fatcat + + A scalable, versioned, API-oriented catalog of bibliographic entities and file metadata # noqa: E501 + + OpenAPI spec version: 0.1.0 + + Generated by: https://github.com/swagger-api/swagger-codegen.git +""" + + +from __future__ import absolute_import + +import unittest + +import fatcat_client +from fatcat_client.models.auth_oidc import AuthOidc # noqa: E501 +from fatcat_client.rest import ApiException + + +class TestAuthOidc(unittest.TestCase): + """AuthOidc unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testAuthOidc(self): + """Test AuthOidc""" + # FIXME: construct object with mandatory attributes with example values + # model = fatcat_client.models.auth_oidc.AuthOidc() # noqa: E501 + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/python/tests/codegen_tests/test_auth_oidc_result.py b/python/tests/codegen_tests/test_auth_oidc_result.py new file mode 100644 index 00000000..d99ef446 --- /dev/null +++ b/python/tests/codegen_tests/test_auth_oidc_result.py @@ -0,0 +1,40 @@ +# coding: utf-8 + +""" + fatcat + + A scalable, versioned, API-oriented catalog of bibliographic entities and file metadata # noqa: E501 + + OpenAPI spec version: 0.1.0 + + Generated by: https://github.com/swagger-api/swagger-codegen.git +""" + + +from __future__ import absolute_import + +import unittest + +import fatcat_client +from fatcat_client.models.auth_oidc_result import AuthOidcResult # noqa: E501 +from fatcat_client.rest import ApiException + + +class TestAuthOidcResult(unittest.TestCase): + """AuthOidcResult unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testAuthOidcResult(self): + """Test AuthOidcResult""" + # FIXME: construct object with mandatory attributes with example values + # model = fatcat_client.models.auth_oidc_result.AuthOidcResult() # noqa: E501 + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/python/tests/codegen_tests/test_default_api.py b/python/tests/codegen_tests/test_default_api.py index ebc66cda..9a632824 100644 --- a/python/tests/codegen_tests/test_default_api.py +++ b/python/tests/codegen_tests/test_default_api.py @@ -35,6 +35,18 @@ class TestDefaultApi(unittest.TestCase): """ pass + def test_auth_check(self): + """Test case for auth_check + + """ + pass + + def test_auth_oidc(self): + """Test case for auth_oidc + + """ + pass + def test_create_container(self): """Test case for create_container @@ -515,6 +527,12 @@ class TestDefaultApi(unittest.TestCase): """ pass + def test_update_editor(self): + """Test case for update_editor + + """ + pass + def test_update_file(self): """Test case for update_file diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py index 567e9132..6a880c48 100644 --- a/python/tests/fixtures.py +++ b/python/tests/fixtures.py @@ -4,12 +4,14 @@ import time import json import signal import pytest +from dotenv import load_dotenv import fatcat_web import fatcat_client @pytest.fixture def full_app(): + load_dotenv(dotenv_path="./env.example") fatcat_web.app.testing = True fatcat_web.app.debug = False return fatcat_web.app @@ -20,8 +22,11 @@ def app(full_app): @pytest.fixture def api(): + load_dotenv(dotenv_path="./env.example") conf = fatcat_client.Configuration() conf.host = "http://localhost:9411/v0" + conf.api_key["Authorization"] = os.getenv("FATCAT_API_AUTH_TOKEN") + conf.api_key_prefix["Authorization"] = "Bearer" api_client = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf)) return api_client diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py index 1fb4a70f..e2ca6122 100644 --- a/python/tests/import_crossref.py +++ b/python/tests/import_crossref.py @@ -2,17 +2,18 @@ import json import pytest from fatcat_tools.importers import CrossrefImporter +from fixtures import api @pytest.fixture(scope="function") -def crossref_importer(): +def crossref_importer(api): with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: - yield CrossrefImporter("http://localhost:9411/v0", issn_file, 'tests/files/example_map.sqlite3', check_existing=False) + yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', check_existing=False) @pytest.fixture(scope="function") -def crossref_importer_existing(): +def crossref_importer_existing(api): with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: - yield CrossrefImporter("http://localhost:9411/v0", issn_file, 'tests/files/example_map.sqlite3', check_existing=True) + yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', check_existing=True) def test_crossref_importer_batch(crossref_importer): with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f: @@ -21,6 +22,13 @@ def test_crossref_importer_batch(crossref_importer): def test_crossref_importer(crossref_importer): with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f: crossref_importer.process_source(f) + # fetch most recent editgroup + changes = crossref_importer.api.get_changelog(limit=1) + eg = changes[0].editgroup + assert eg.description + assert "crossref" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.CrossrefImporter" in eg.extra['agent'] def test_crossref_mappings(crossref_importer): assert crossref_importer.map_release_type('journal-article') == "article-journal" diff --git a/python/tests/import_grobid_metadata.py b/python/tests/import_grobid_metadata.py index 459b247b..97ebcaef 100644 --- a/python/tests/import_grobid_metadata.py +++ b/python/tests/import_grobid_metadata.py @@ -4,6 +4,7 @@ import json import base64 import pytest from fatcat_tools.importers import GrobidMetadataImporter +from fixtures import api """ WARNING: these tests are currently very fragile because they have database @@ -11,8 +12,8 @@ side-effects. Should probably be disabled or re-written. """ @pytest.fixture(scope="function") -def grobid_metadata_importer(): - yield GrobidMetadataImporter("http://localhost:9411/v0") +def grobid_metadata_importer(api): + yield GrobidMetadataImporter(api) # TODO: use API to check that entities actually created... #def test_grobid_metadata_importer_batch(grobid_metadata_importer): @@ -54,3 +55,11 @@ def test_file_metadata_parse(grobid_metadata_importer): def test_grobid_metadata_importer(grobid_metadata_importer): with open('tests/files/example_grobid_metadata_lines.tsv', 'r') as f: grobid_metadata_importer.process_source(f) + + # fetch most recent editgroup + changes = grobid_metadata_importer.api.get_changelog(limit=1) + eg = changes[0].editgroup + assert eg.description + assert "grobid" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.GrobidMetadataImporter" in eg.extra['agent'] diff --git a/python/tests/import_issn.py b/python/tests/import_issn.py index 98a9f4a7..6b5978d9 100644 --- a/python/tests/import_issn.py +++ b/python/tests/import_issn.py @@ -1,11 +1,12 @@ import pytest from fatcat_tools.importers import IssnImporter +from fixtures import api @pytest.fixture(scope="function") -def issn_importer(): - yield IssnImporter("http://localhost:9411/v0") +def issn_importer(api): + yield IssnImporter(api) # TODO: use API to check that entities actually created... def test_issn_importer_batch(issn_importer): @@ -15,3 +16,11 @@ def test_issn_importer_batch(issn_importer): def test_issn_importer(issn_importer): with open('tests/files/journal_extra_metadata.snip.csv', 'r') as f: issn_importer.process_csv_source(f) + + # fetch most recent editgroup + changes = issn_importer.api.get_changelog(limit=1) + eg = changes[0].editgroup + assert eg.description + assert "container" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.IssnImporter" in eg.extra['agent'] diff --git a/python/tests/import_matched.py b/python/tests/import_matched.py index 46a9ef85..080674ac 100644 --- a/python/tests/import_matched.py +++ b/python/tests/import_matched.py @@ -2,11 +2,12 @@ import json import pytest from fatcat_tools.importers import MatchedImporter +from fixtures import api @pytest.fixture(scope="function") -def matched_importer(): - yield MatchedImporter("http://localhost:9411/v0") +def matched_importer(api): + yield MatchedImporter(api) # TODO: use API to check that entities actually created... def test_matched_importer_batch(matched_importer): @@ -17,6 +18,14 @@ def test_matched_importer(matched_importer): with open('tests/files/example_matched.json', 'r') as f: matched_importer.process_source(f) + # fetch most recent editgroup + changes = matched_importer.api.get_changelog(limit=1) + eg = changes[0].editgroup + assert eg.description + assert "file-to-release" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.MatchedImporter" in eg.extra['agent'] + def test_matched_dict_parse(matched_importer): with open('tests/files/example_matched.json', 'r') as f: raw = json.loads(f.readline()) diff --git a/python/tests/import_orcid.py b/python/tests/import_orcid.py index 18199888..717a1328 100644 --- a/python/tests/import_orcid.py +++ b/python/tests/import_orcid.py @@ -2,11 +2,12 @@ import json import pytest from fatcat_tools.importers import OrcidImporter +from fixtures import api @pytest.fixture(scope="function") -def orcid_importer(): - yield OrcidImporter("http://localhost:9411/v0") +def orcid_importer(api): + yield OrcidImporter(api) # TODO: use API to check that entities actually created... def test_orcid_importer_batch(orcid_importer): @@ -21,6 +22,14 @@ def test_orcid_importer(orcid_importer): with open('tests/files/0000-0001-8254-7103.json', 'r') as f: orcid_importer.process_source(f) + # fetch most recent editgroup + changes = orcid_importer.api.get_changelog(limit=1) + eg = changes[0].editgroup + assert eg.description + assert "orcid" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.OrcidImporter" in eg.extra['agent'] + def test_orcid_importer_x(orcid_importer): with open('tests/files/0000-0003-3953-765X.json', 'r') as f: orcid_importer.process_source(f) diff --git a/python/tests/importer.py b/python/tests/importer.py index f228a9b2..34efa5d8 100644 --- a/python/tests/importer.py +++ b/python/tests/importer.py @@ -2,11 +2,12 @@ import pytest from fatcat_tools.importers import FatcatImporter +from fixtures import api -def test_issnl_mapping_lookup(): +def test_issnl_mapping_lookup(api): with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: - fi = FatcatImporter("http://localhost:9411/v0", issn_file) + fi = FatcatImporter(api, issn_map_file=issn_file) assert fi.issn2issnl('0000-0027') == '0002-0027' assert fi.issn2issnl('0002-0027') == '0002-0027' @@ -14,10 +15,10 @@ def test_issnl_mapping_lookup(): assert fi.lookup_issnl('9999-9999') == None -def test_identifiers(): +def test_identifiers(api): with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: - fi = FatcatImporter("http://localhost:9411/v0", issn_file) + fi = FatcatImporter(api, issn_map_file=issn_file) assert fi.is_issnl("1234-5678") == True assert fi.is_issnl("1234-5678.") == False diff --git a/python/tests/transform_tests.py b/python/tests/transform_tests.py index a42db244..e9d23250 100644 --- a/python/tests/transform_tests.py +++ b/python/tests/transform_tests.py @@ -3,6 +3,7 @@ import json import pytest from fatcat_tools import * from fatcat_client import * +from fixtures import api from import_crossref import crossref_importer |