aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/fatcat_tools/workers/elasticsearch.py13
-rwxr-xr-xpython/fatcat_transform.py22
-rw-r--r--python/fatcat_web/forms.py26
-rw-r--r--python/tests/import_crossref.py3
-rw-r--r--python/tests/transform_csl.py8
-rw-r--r--python/tests/transform_elasticsearch.py31
-rw-r--r--python/tests/transform_toml.py7
7 files changed, 76 insertions, 34 deletions
diff --git a/python/fatcat_tools/workers/elasticsearch.py b/python/fatcat_tools/workers/elasticsearch.py
index b8735a37..91f2796c 100644
--- a/python/fatcat_tools/workers/elasticsearch.py
+++ b/python/fatcat_tools/workers/elasticsearch.py
@@ -1,17 +1,20 @@
-import sys
import json
+import sys
-import requests
import elasticsearch
+import requests
from confluent_kafka import Consumer, KafkaException
+from fatcat_openapi_client import ApiClient, ChangelogEntry, ContainerEntity, ReleaseEntity
-from fatcat_openapi_client import ReleaseEntity, ContainerEntity, ApiClient, ChangelogEntry
-from fatcat_tools import (public_api, entity_from_json,
- release_to_elasticsearch, container_to_elasticsearch,
+from fatcat_tools import entity_from_json, public_api
+from fatcat_tools.transforms import (
changelog_to_elasticsearch,
+ container_to_elasticsearch,
+ release_to_elasticsearch,
)
from fatcat_web.search import get_elastic_container_stats
+
from .worker_common import FatcatWorker
diff --git a/python/fatcat_transform.py b/python/fatcat_transform.py
index 93c39e2f..ab855dbf 100755
--- a/python/fatcat_transform.py
+++ b/python/fatcat_transform.py
@@ -5,17 +5,23 @@ Utility script for doing bulk conversion/transforms of entity JSON schema to
other formats
"""
-import sys
-import json
import argparse
+import json
+import sys
import elasticsearch
-from fatcat_openapi_client import ReleaseEntity, ContainerEntity, FileEntity, ChangelogEntry
-
-from fatcat_tools import entity_from_json, \
- release_to_elasticsearch, container_to_elasticsearch, \
- file_to_elasticsearch, changelog_to_elasticsearch, public_api, \
- release_to_csl, citeproc_csl
+from fatcat_openapi_client import ChangelogEntry, ContainerEntity, FileEntity, ReleaseEntity
+
+from fatcat_tools import public_api
+from fatcat_tools.transforms import (
+ changelog_to_elasticsearch,
+ citeproc_csl,
+ container_to_elasticsearch,
+ entity_from_json,
+ file_to_elasticsearch,
+ release_to_csl,
+ release_to_elasticsearch,
+)
from fatcat_web.search import get_elastic_container_stats
diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py
index 2757ebd2..b432ac16 100644
--- a/python/fatcat_web/forms.py
+++ b/python/fatcat_web/forms.py
@@ -7,13 +7,29 @@ but can't find one that is actually maintained.
import datetime
import toml
+from fatcat_openapi_client import (
+ ContainerEntity,
+ FileEntity,
+ FileUrl,
+ ReleaseContrib,
+ ReleaseEntity,
+ ReleaseExtIds,
+)
from flask_wtf import FlaskForm
-from wtforms import SelectField, DateField, StringField, IntegerField, \
- HiddenField, FormField, FieldList, validators, ValidationError, TextAreaField
+from wtforms import (
+ DateField,
+ FieldList,
+ FormField,
+ HiddenField,
+ IntegerField,
+ SelectField,
+ StringField,
+ TextAreaField,
+ ValidationError,
+ validators,
+)
-from fatcat_tools import entity_to_toml
-from fatcat_openapi_client import ContainerEntity, FileEntity, \
- ReleaseEntity, ReleaseContrib, FileUrl, ReleaseExtIds
+from fatcat_tools.transforms import entity_to_toml
release_type_options = [
('', 'Unknown (blank)'),
diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py
index 65cd2c37..ebe6942f 100644
--- a/python/tests/import_crossref.py
+++ b/python/tests/import_crossref.py
@@ -4,7 +4,8 @@ import gzip
import pytest
from fatcat_tools.importers import CrossrefImporter, JsonLinePusher
-from fixtures import *
+
+from fixtures import api
@pytest.fixture(scope="function")
diff --git a/python/tests/transform_csl.py b/python/tests/transform_csl.py
index 77ce1bff..54e85f91 100644
--- a/python/tests/transform_csl.py
+++ b/python/tests/transform_csl.py
@@ -1,11 +1,13 @@
import json
-import pytest
-from fatcat_tools import *
+import pytest
from fatcat_openapi_client import *
-from import_crossref import crossref_importer
from fixtures import *
+from import_crossref import crossref_importer
+
+from fatcat_tools.transforms import citeproc_csl, entity_from_json, release_to_csl
+
def test_csl_crossref(crossref_importer):
with open('tests/files/crossref-works.single.json', 'r') as f:
diff --git a/python/tests/transform_elasticsearch.py b/python/tests/transform_elasticsearch.py
index ba2b7ea2..b9011892 100644
--- a/python/tests/transform_elasticsearch.py
+++ b/python/tests/transform_elasticsearch.py
@@ -1,13 +1,26 @@
-
-import json
import datetime
+import json
-from fatcat_tools import *
-from fatcat_openapi_client import *
-from import_journal_metadata import journal_metadata_importer
+from fatcat_openapi_client import (
+ ChangelogEntry,
+ ContainerEntity,
+ FileEntity,
+ FileUrl,
+ ReleaseEntity,
+ ReleaseExtIds,
+ ReleaseRef,
+)
+from fixtures import api
from import_crossref import crossref_importer
-from import_matched import matched_importer
-from fixtures import *
+from import_journal_metadata import journal_metadata_importer
+
+from fatcat_tools.transforms import (
+ changelog_to_elasticsearch,
+ container_to_elasticsearch,
+ entity_from_json,
+ file_to_elasticsearch,
+ release_to_elasticsearch,
+)
def test_basic_elasticsearch_convert(crossref_importer):
@@ -190,7 +203,7 @@ def test_elasticsearch_container_transform(journal_metadata_importer):
assert es['any_kbart'] == True
-def test_elasticsearch_file_transform(matched_importer):
+def test_elasticsearch_file_transform():
f = entity_from_json(open('./tests/files/file_bcah4zp5tvdhjl5bqci2c2lgfa.json', 'r').read(), FileEntity)
f.state = 'active'
@@ -210,7 +223,7 @@ def test_elasticsearch_file_transform(matched_importer):
# old regression
assert not '.archive.org' in (es['hosts'] + es['domains'])
-def test_elasticsearch_changelog_transform(matched_importer):
+def test_elasticsearch_changelog_transform():
ce = entity_from_json(open('./tests/files/changelog_3469683.json', 'r').read(), ChangelogEntry)
es = changelog_to_elasticsearch(ce)
diff --git a/python/tests/transform_toml.py b/python/tests/transform_toml.py
index d12ba027..29da1d38 100644
--- a/python/tests/transform_toml.py
+++ b/python/tests/transform_toml.py
@@ -1,10 +1,11 @@
import json
-from fatcat_tools import *
-from fatcat_openapi_client import *
+from fatcat_openapi_client import ReleaseEntity
+from fixtures import api
from import_crossref import crossref_importer
-from fixtures import *
+
+from fatcat_tools.transforms import entity_from_toml, entity_to_toml
def test_basic_toml(crossref_importer):