summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xpython/fatcat_cleanup.py6
-rwxr-xr-xpython/fatcat_import.py43
-rwxr-xr-xpython/fatcat_tools/importers/cdl_dash_dat.py21
-rw-r--r--python/fatcat_tools/importers/matched.py5
-rw-r--r--python/fatcat_tools/importers/pubmed.py12
-rw-r--r--python/fatcat_tools/importers/shadow.py5
-rwxr-xr-xpython/fatcat_tools/importers/wayback_static.py19
-rw-r--r--python/fatcat_web/editing_routes.py32
-rw-r--r--python/fatcat_web/entity_helpers.py21
-rw-r--r--python/fatcat_web/ref_routes.py24
-rw-r--r--python/fatcat_web/routes.py77
-rw-r--r--python/tests/transform_csl.py4
12 files changed, 207 insertions, 62 deletions
diff --git a/python/fatcat_cleanup.py b/python/fatcat_cleanup.py
index 4e11139e..04ee2bd9 100755
--- a/python/fatcat_cleanup.py
+++ b/python/fatcat_cleanup.py
@@ -1,14 +1,14 @@
#!/usr/bin/env python3
+import argparse
import os
import sys
-import argparse
+
import raven
from fatcat_tools import authenticated_api
+from fatcat_tools.cleanups import FileCleaner
from fatcat_tools.importers import JsonLinePusher
-from fatcat_tools.cleanups import *
-
# Yep, a global. Gets DSN from `SENTRY_DSN` environment variable
sentry_client = raven.Client()
diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index 41a51ad4..3225688c 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -1,13 +1,50 @@
#!/usr/bin/env python3
+import argparse
import os
import sys
-import argparse
+
import raven
from fatcat_tools import authenticated_api
-from fatcat_tools.importers import *
-
+from fatcat_tools.importers import (
+ ARABESQUE_MATCH_WHERE_CLAUSE,
+ ArabesqueMatchImporter,
+ ArxivRawImporter,
+ Bs4XmlFileListPusher,
+ Bs4XmlFilePusher,
+ Bs4XmlLargeFilePusher,
+ Bs4XmlLinesPusher,
+ ChoculaImporter,
+ CrossrefImporter,
+ DataciteImporter,
+ DblpContainerImporter,
+ DblpReleaseImporter,
+ DoajArticleImporter,
+ FileMetaImporter,
+ FilesetImporter,
+ GrobidMetadataImporter,
+ IngestFileResultImporter,
+ IngestFilesetResultImporter,
+ IngestWebResultImporter,
+ JalcImporter,
+ JournalMetadataImporter,
+ JsonLinePusher,
+ JstorImporter,
+ KafkaBs4XmlPusher,
+ KafkaJsonPusher,
+ LinePusher,
+ MatchedImporter,
+ OrcidImporter,
+ PubmedImporter,
+ SavePaperNowFileImporter,
+ SavePaperNowFilesetImporter,
+ SavePaperNowWebImporter,
+ ShadowLibraryImporter,
+ SqlitePusher,
+ auto_cdl_dash_dat,
+ auto_wayback_static,
+)
# Yep, a global. Gets DSN from `SENTRY_DSN` environment variable
sentry_client = raven.Client()
diff --git a/python/fatcat_tools/importers/cdl_dash_dat.py b/python/fatcat_tools/importers/cdl_dash_dat.py
index 36a2f9a6..0340f6a3 100755
--- a/python/fatcat_tools/importers/cdl_dash_dat.py
+++ b/python/fatcat_tools/importers/cdl_dash_dat.py
@@ -1,16 +1,25 @@
#!/usr/bin/env python3
-import os
-import sys
-import json
-import magic
-import urllib
import hashlib
+import json
import mimetypes
+import os
import subprocess
+import sys
+import urllib
import fatcat_openapi_client
-from fatcat_openapi_client import *
+import magic
+from fatcat_openapi_client import (
+ Editgroup,
+ FilesetEntity,
+ FilesetFile,
+ ReleaseAbstract,
+ ReleaseContrib,
+ ReleaseEntity,
+ ReleaseExtIds,
+)
+
from .common import clean
from .crossref import lookup_license_slug
diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py
index d95c5847..e0e4fc3c 100644
--- a/python/fatcat_tools/importers/matched.py
+++ b/python/fatcat_tools/importers/matched.py
@@ -1,8 +1,9 @@
import fatcat_openapi_client
-from fatcat_tools.normal import *
-from .common import EntityImporter, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS
+from fatcat_tools.normal import clean_doi
+
+from .common import SANE_MAX_RELEASES, SANE_MAX_URLS, EntityImporter, make_rel_url
class MatchedImporter(EntityImporter):
diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py
index c34fd7d6..c9907c5e 100644
--- a/python/fatcat_tools/importers/pubmed.py
+++ b/python/fatcat_tools/importers/pubmed.py
@@ -1,13 +1,15 @@
-import sys
-import json
import datetime
+import json
+import sys
import warnings
-from bs4 import BeautifulSoup
import fatcat_openapi_client
-from fatcat_tools.normal import *
-from .common import EntityImporter, clean, LANG_MAP_MARC
+from bs4 import BeautifulSoup
+
+from fatcat_tools.normal import clean_doi, clean_issn, clean_pmcid, clean_pmid
+
+from .common import LANG_MAP_MARC, EntityImporter, clean
# from: https://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.publication_types/?report=objectonly
PUBMED_RELEASE_TYPE_MAP = {
diff --git a/python/fatcat_tools/importers/shadow.py b/python/fatcat_tools/importers/shadow.py
index c04e9aa8..fa9b4d10 100644
--- a/python/fatcat_tools/importers/shadow.py
+++ b/python/fatcat_tools/importers/shadow.py
@@ -1,8 +1,9 @@
import fatcat_openapi_client
-from fatcat_tools.normal import *
-from .common import EntityImporter, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS
+from fatcat_tools.normal import clean_doi, clean_isbn13, clean_pmid
+
+from .common import SANE_MAX_RELEASES, SANE_MAX_URLS, EntityImporter, make_rel_url
class ShadowLibraryImporter(EntityImporter):
diff --git a/python/fatcat_tools/importers/wayback_static.py b/python/fatcat_tools/importers/wayback_static.py
index 5b23347f..196f86ff 100755
--- a/python/fatcat_tools/importers/wayback_static.py
+++ b/python/fatcat_tools/importers/wayback_static.py
@@ -6,16 +6,23 @@ Helpers to create Web Capture entities from extracted wayback content.
Works as a stand-alone script (for debugging) or as library routines.
"""
-import sys
-import json
-import hashlib
-import requests
-import datetime
import argparse
+import datetime
+import hashlib
+import json
import subprocess
+import sys
+
+import requests
from bs4 import BeautifulSoup
+from fatcat_openapi_client import (
+ ApiClient,
+ Editgroup,
+ WebcaptureCdxLine,
+ WebcaptureEntity,
+ WebcaptureUrl,
+)
-from fatcat_openapi_client import *
from .common import b32_hex
CDX_API_BASE = "https://web.archive.org/cdx/search/cdx"
diff --git a/python/fatcat_web/editing_routes.py b/python/fatcat_web/editing_routes.py
index 61aade72..d888735a 100644
--- a/python/fatcat_web/editing_routes.py
+++ b/python/fatcat_web/editing_routes.py
@@ -1,17 +1,31 @@
from typing import Optional
-from flask import render_template, abort, redirect, session, flash
-from flask_login import login_required
-
-from fatcat_openapi_client import *
+from fatcat_openapi_client import (
+ ContainerEntity,
+ CreatorEntity,
+ Editgroup,
+ EntityEdit,
+ FileEntity,
+ FilesetEntity,
+ ReleaseEntity,
+ WebcaptureEntity,
+ WorkEntity,
+)
from fatcat_openapi_client.rest import ApiException
-from fatcat_tools.transforms import *
-from fatcat_web import app, api, auth_api
-from fatcat_web.search import *
-from fatcat_web.forms import *
-from fatcat_web.entity_helpers import *
+from flask import abort, flash, redirect, render_template, session
+from flask_login import login_required
+from fatcat_tools.transforms import entity_from_toml
+from fatcat_web import api, app, auth_api
+from fatcat_web.entity_helpers import generic_get_editgroup_entity, generic_get_entity
+from fatcat_web.forms import (
+ ContainerEntityForm,
+ EntityEditForm,
+ EntityTomlForm,
+ FileEntityForm,
+ ReleaseEntityForm,
+)
### Helper Methods ##########################################################
diff --git a/python/fatcat_web/entity_helpers.py b/python/fatcat_web/entity_helpers.py
index ab4c16b5..26371341 100644
--- a/python/fatcat_web/entity_helpers.py
+++ b/python/fatcat_web/entity_helpers.py
@@ -1,11 +1,26 @@
-from flask import abort
-from fatcat_openapi_client import *
+from fatcat_openapi_client import (
+ ContainerEntity,
+ CreatorEntity,
+ FileEntity,
+ FilesetEntity,
+ ReleaseEntity,
+ ReleaseExtIds,
+ WebcaptureEntity,
+ WorkEntity,
+)
from fatcat_openapi_client.rest import ApiException, ApiValueError
-from fatcat_tools.transforms import *
+from flask import abort
+
+from fatcat_tools.transforms import (
+ container_to_elasticsearch,
+ file_to_elasticsearch,
+ release_to_elasticsearch,
+)
from fatcat_web import api
from fatcat_web.hacks import strip_extlink_xml, wayback_suffix
+
def enrich_container_entity(entity):
if entity.state in ('redirect', 'deleted'):
return entity
diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py
index 2d8ed413..eed3f1df 100644
--- a/python/fatcat_web/ref_routes.py
+++ b/python/fatcat_web/ref_routes.py
@@ -5,18 +5,28 @@ Flask endpoints for reference (citation) endpoints. Eg, listing references
import json
-from flask import render_template, request, Response, jsonify
-from fatcat_openapi_client import *
-from fuzzycat.grobid_unstructured import grobid_api_process_citation, transform_grobid_ref_xml, grobid_ref_to_release
+from flask import Response, jsonify, render_template, request
+from fuzzycat.grobid_unstructured import (
+ grobid_api_process_citation,
+ grobid_ref_to_release,
+ transform_grobid_ref_xml,
+)
from fuzzycat.simple import close_fuzzy_biblio_matches, close_fuzzy_release_matches
-from fatcat_tools.references import enrich_inbound_refs, enrich_outbound_refs, get_inbound_refs, get_outbound_refs, RefHits
+from fatcat_tools.references import (
+ RefHits,
+ enrich_inbound_refs,
+ enrich_outbound_refs,
+ get_inbound_refs,
+ get_outbound_refs,
+)
from fatcat_tools.transforms.access import release_access_options
from fatcat_tools.transforms.entities import entity_to_dict
-from fatcat_web import app, api
+from fatcat_web import api, app
from fatcat_web.cors import crossdomain
-from fatcat_web.forms import *
-from fatcat_web.entity_helpers import *
+from fatcat_web.entity_helpers import generic_get_entity
+from fatcat_web.forms import ReferenceMatchForm
+
def _refs_web(direction, release_ident=None, work_ident=None, openlibrary_id=None, wikipedia_article=None) -> RefHits:
offset = request.args.get('offset', '0')
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index e9741b6d..9795adf7 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -1,25 +1,74 @@
-import os
import json
+import os
+
import citeproc_styles
-from flask import render_template, make_response, send_from_directory, \
- request, url_for, abort, redirect, jsonify, session, Response
+from fatcat_openapi_client import EditgroupAnnotation
+from fatcat_openapi_client.rest import ApiException, ApiValueError
+from flask import (
+ Response,
+ abort,
+ jsonify,
+ make_response,
+ redirect,
+ render_template,
+ request,
+ send_from_directory,
+ session,
+ url_for,
+)
from flask_login import login_required
from flask_wtf.csrf import CSRFError
-from fatcat_openapi_client import EditgroupAnnotation
-from fatcat_openapi_client.rest import ApiException
-from fatcat_tools.transforms import *
-from fatcat_tools.normal import *
-from fatcat_web import app, api, auth_api, priv_api, mwoauth, Config
-from fatcat_web.auth import handle_token_login, handle_logout, load_user, handle_ia_xauth, handle_wmoauth
+from fatcat_tools.normal import (
+ clean_arxiv_id,
+ clean_doi,
+ clean_isbn13,
+ clean_issn,
+ clean_orcid,
+ clean_pmcid,
+ clean_sha1,
+ clean_sha256,
+)
+from fatcat_tools.transforms import citeproc_csl, release_to_csl
+from fatcat_web import Config, api, app, auth_api, mwoauth, priv_api
+from fatcat_web.auth import (
+ handle_ia_xauth,
+ handle_logout,
+ handle_token_login,
+ handle_wmoauth,
+ load_user,
+)
from fatcat_web.cors import crossdomain
-from fatcat_web.search import *
-from fatcat_web.entity_helpers import *
-from fatcat_web.graphics import *
-from fatcat_web.kafka import *
+from fatcat_web.entity_helpers import (
+ generic_get_editgroup_entity,
+ generic_get_entity,
+ generic_get_entity_revision,
+)
from fatcat_web.forms import SavePaperNowForm
-
+from fatcat_web.graphics import (
+ ia_coverage_histogram,
+ preservation_by_date_histogram,
+ preservation_by_volume_histogram,
+ preservation_by_year_histogram,
+)
+from fatcat_web.kafka import kafka_pixy_produce
+from fatcat_web.search import (
+ FatcatSearchError,
+ GenericQuery,
+ ReleaseQuery,
+ do_container_search,
+ do_release_search,
+ get_elastic_container_histogram_legacy,
+ get_elastic_container_preservation_by_volume,
+ get_elastic_container_random_releases,
+ get_elastic_container_stats,
+ get_elastic_entity_stats,
+ get_elastic_preservation_by_date,
+ get_elastic_preservation_by_type,
+ get_elastic_preservation_by_year,
+ get_elastic_search_coverage,
+)
### Generic Entity Views ####################################################
diff --git a/python/tests/transform_csl.py b/python/tests/transform_csl.py
index 54e85f91..83266669 100644
--- a/python/tests/transform_csl.py
+++ b/python/tests/transform_csl.py
@@ -2,8 +2,8 @@
import json
import pytest
-from fatcat_openapi_client import *
-from fixtures import *
+from fatcat_openapi_client import ReleaseEntity
+from fixtures import api
from import_crossref import crossref_importer
from fatcat_tools.transforms import citeproc_csl, entity_from_json, release_to_csl