diff options
-rw-r--r-- | python/fatcat_tools/workers/elasticsearch.py | 13 | ||||
-rwxr-xr-x | python/fatcat_transform.py | 22 | ||||
-rw-r--r-- | python/fatcat_web/forms.py | 26 | ||||
-rw-r--r-- | python/tests/import_crossref.py | 3 | ||||
-rw-r--r-- | python/tests/transform_csl.py | 8 | ||||
-rw-r--r-- | python/tests/transform_elasticsearch.py | 31 | ||||
-rw-r--r-- | python/tests/transform_toml.py | 7 |
7 files changed, 76 insertions, 34 deletions
diff --git a/python/fatcat_tools/workers/elasticsearch.py b/python/fatcat_tools/workers/elasticsearch.py index b8735a37..91f2796c 100644 --- a/python/fatcat_tools/workers/elasticsearch.py +++ b/python/fatcat_tools/workers/elasticsearch.py @@ -1,17 +1,20 @@ -import sys import json +import sys -import requests import elasticsearch +import requests from confluent_kafka import Consumer, KafkaException +from fatcat_openapi_client import ApiClient, ChangelogEntry, ContainerEntity, ReleaseEntity -from fatcat_openapi_client import ReleaseEntity, ContainerEntity, ApiClient, ChangelogEntry -from fatcat_tools import (public_api, entity_from_json, - release_to_elasticsearch, container_to_elasticsearch, +from fatcat_tools import entity_from_json, public_api +from fatcat_tools.transforms import ( changelog_to_elasticsearch, + container_to_elasticsearch, + release_to_elasticsearch, ) from fatcat_web.search import get_elastic_container_stats + from .worker_common import FatcatWorker diff --git a/python/fatcat_transform.py b/python/fatcat_transform.py index 93c39e2f..ab855dbf 100755 --- a/python/fatcat_transform.py +++ b/python/fatcat_transform.py @@ -5,17 +5,23 @@ Utility script for doing bulk conversion/transforms of entity JSON schema to other formats """ -import sys -import json import argparse +import json +import sys import elasticsearch -from fatcat_openapi_client import ReleaseEntity, ContainerEntity, FileEntity, ChangelogEntry - -from fatcat_tools import entity_from_json, \ - release_to_elasticsearch, container_to_elasticsearch, \ - file_to_elasticsearch, changelog_to_elasticsearch, public_api, \ - release_to_csl, citeproc_csl +from fatcat_openapi_client import ChangelogEntry, ContainerEntity, FileEntity, ReleaseEntity + +from fatcat_tools import public_api +from fatcat_tools.transforms import ( + changelog_to_elasticsearch, + citeproc_csl, + container_to_elasticsearch, + entity_from_json, + file_to_elasticsearch, + release_to_csl, + release_to_elasticsearch, +) from fatcat_web.search import get_elastic_container_stats diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py index 2757ebd2..b432ac16 100644 --- a/python/fatcat_web/forms.py +++ b/python/fatcat_web/forms.py @@ -7,13 +7,29 @@ but can't find one that is actually maintained. import datetime import toml +from fatcat_openapi_client import ( + ContainerEntity, + FileEntity, + FileUrl, + ReleaseContrib, + ReleaseEntity, + ReleaseExtIds, +) from flask_wtf import FlaskForm -from wtforms import SelectField, DateField, StringField, IntegerField, \ - HiddenField, FormField, FieldList, validators, ValidationError, TextAreaField +from wtforms import ( + DateField, + FieldList, + FormField, + HiddenField, + IntegerField, + SelectField, + StringField, + TextAreaField, + ValidationError, + validators, +) -from fatcat_tools import entity_to_toml -from fatcat_openapi_client import ContainerEntity, FileEntity, \ - ReleaseEntity, ReleaseContrib, FileUrl, ReleaseExtIds +from fatcat_tools.transforms import entity_to_toml release_type_options = [ ('', 'Unknown (blank)'), diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py index 65cd2c37..ebe6942f 100644 --- a/python/tests/import_crossref.py +++ b/python/tests/import_crossref.py @@ -4,7 +4,8 @@ import gzip import pytest from fatcat_tools.importers import CrossrefImporter, JsonLinePusher -from fixtures import * + +from fixtures import api @pytest.fixture(scope="function") diff --git a/python/tests/transform_csl.py b/python/tests/transform_csl.py index 77ce1bff..54e85f91 100644 --- a/python/tests/transform_csl.py +++ b/python/tests/transform_csl.py @@ -1,11 +1,13 @@ import json -import pytest -from fatcat_tools import * +import pytest from fatcat_openapi_client import * -from import_crossref import crossref_importer from fixtures import * +from import_crossref import crossref_importer + +from fatcat_tools.transforms import citeproc_csl, entity_from_json, release_to_csl + def test_csl_crossref(crossref_importer): with open('tests/files/crossref-works.single.json', 'r') as f: diff --git a/python/tests/transform_elasticsearch.py b/python/tests/transform_elasticsearch.py index ba2b7ea2..b9011892 100644 --- a/python/tests/transform_elasticsearch.py +++ b/python/tests/transform_elasticsearch.py @@ -1,13 +1,26 @@ - -import json import datetime +import json -from fatcat_tools import * -from fatcat_openapi_client import * -from import_journal_metadata import journal_metadata_importer +from fatcat_openapi_client import ( + ChangelogEntry, + ContainerEntity, + FileEntity, + FileUrl, + ReleaseEntity, + ReleaseExtIds, + ReleaseRef, +) +from fixtures import api from import_crossref import crossref_importer -from import_matched import matched_importer -from fixtures import * +from import_journal_metadata import journal_metadata_importer + +from fatcat_tools.transforms import ( + changelog_to_elasticsearch, + container_to_elasticsearch, + entity_from_json, + file_to_elasticsearch, + release_to_elasticsearch, +) def test_basic_elasticsearch_convert(crossref_importer): @@ -190,7 +203,7 @@ def test_elasticsearch_container_transform(journal_metadata_importer): assert es['any_kbart'] == True -def test_elasticsearch_file_transform(matched_importer): +def test_elasticsearch_file_transform(): f = entity_from_json(open('./tests/files/file_bcah4zp5tvdhjl5bqci2c2lgfa.json', 'r').read(), FileEntity) f.state = 'active' @@ -210,7 +223,7 @@ def test_elasticsearch_file_transform(matched_importer): # old regression assert not '.archive.org' in (es['hosts'] + es['domains']) -def test_elasticsearch_changelog_transform(matched_importer): +def test_elasticsearch_changelog_transform(): ce = entity_from_json(open('./tests/files/changelog_3469683.json', 'r').read(), ChangelogEntry) es = changelog_to_elasticsearch(ce) diff --git a/python/tests/transform_toml.py b/python/tests/transform_toml.py index d12ba027..29da1d38 100644 --- a/python/tests/transform_toml.py +++ b/python/tests/transform_toml.py @@ -1,10 +1,11 @@ import json -from fatcat_tools import * -from fatcat_openapi_client import * +from fatcat_openapi_client import ReleaseEntity +from fixtures import api from import_crossref import crossref_importer -from fixtures import * + +from fatcat_tools.transforms import entity_from_toml, entity_to_toml def test_basic_toml(crossref_importer): |