diff options
Diffstat (limited to 'python')
85 files changed, 278 insertions, 184 deletions
diff --git a/python/fatcat_export.py b/python/fatcat_export.py index 763c217e..9a7cfd01 100755 --- a/python/fatcat_export.py +++ b/python/fatcat_export.py @@ -7,11 +7,11 @@ These scripts are just a demonstration of how the API *could* be scraped without permission by an third party. """ -import sys -import json import argparse +import json +import sys -from fatcat_tools import uuid2fcid, entity_to_dict, public_api +from fatcat_tools import entity_to_dict, public_api, uuid2fcid def run_export_releases(args): diff --git a/python/fatcat_harvest.py b/python/fatcat_harvest.py index 5f67c679..0324aa52 100755 --- a/python/fatcat_harvest.py +++ b/python/fatcat_harvest.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 -import sys import argparse import datetime +import sys + import raven -from fatcat_tools.harvest import HarvestCrossrefWorker, HarvestDataciteWorker,\ - HarvestArxivWorker, HarvestDoajArticleWorker, HarvestDoajJournalWorker,\ - PubmedFTPWorker + +from fatcat_tools.harvest import ( + HarvestArxivWorker, + HarvestCrossrefWorker, + HarvestDataciteWorker, + HarvestDoajArticleWorker, + HarvestDoajJournalWorker, + PubmedFTPWorker, +) # Yep, a global. Gets DSN from `SENTRY_DSN` environment variable sentry_client = raven.Client() diff --git a/python/fatcat_ingest.py b/python/fatcat_ingest.py index b62cb8a4..165e42f3 100755 --- a/python/fatcat_ingest.py +++ b/python/fatcat_ingest.py @@ -5,18 +5,18 @@ Intended to be a command line interface to "Save Paper Now" and ingest request/response. """ -import sys -import json import argparse +import json +import sys from collections import Counter -import raven + import elasticsearch -from elasticsearch_dsl import Search, Q +import raven +from elasticsearch_dsl import Q, Search -from fatcat_tools import public_api, simple_kafka_producer, kafka_fail_fast +from fatcat_tools import kafka_fail_fast, public_api, simple_kafka_producer from fatcat_tools.transforms import release_ingest_request - # Yep, a global. Gets DSN from `SENTRY_DSN` environment variable sentry_client = raven.Client() diff --git a/python/fatcat_review.py b/python/fatcat_review.py index a10fc34b..f719bb46 100755 --- a/python/fatcat_review.py +++ b/python/fatcat_review.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -import sys import argparse +import sys + import raven from fatcat_tools import authenticated_api diff --git a/python/fatcat_tools/api_auth.py b/python/fatcat_tools/api_auth.py index 13310120..bbf059c0 100644 --- a/python/fatcat_tools/api_auth.py +++ b/python/fatcat_tools/api_auth.py @@ -1,6 +1,7 @@ import os import sys + import fatcat_openapi_client diff --git a/python/fatcat_tools/cleanups/common.py b/python/fatcat_tools/cleanups/common.py index 04e6ade4..d0fcc761 100644 --- a/python/fatcat_tools/cleanups/common.py +++ b/python/fatcat_tools/cleanups/common.py @@ -1,10 +1,11 @@ -import json import copy +import json import subprocess from collections import Counter from fatcat_openapi_client import ApiClient, Editgroup + from fatcat_tools.transforms import entity_from_dict, entity_to_dict diff --git a/python/fatcat_tools/cleanups/files.py b/python/fatcat_tools/cleanups/files.py index 10dd45cc..0d275ba6 100644 --- a/python/fatcat_tools/cleanups/files.py +++ b/python/fatcat_tools/cleanups/files.py @@ -1,6 +1,6 @@ -from fatcat_openapi_client.rest import ApiException from fatcat_openapi_client.models import FileEntity +from fatcat_openapi_client.rest import ApiException from .common import EntityCleaner diff --git a/python/fatcat_tools/fcid.py b/python/fatcat_tools/fcid.py index 4194ea63..0987d10d 100644 --- a/python/fatcat_tools/fcid.py +++ b/python/fatcat_tools/fcid.py @@ -2,6 +2,7 @@ import base64 import uuid + def fcid2uuid(s): """ Converts a fatcat identifier (base32 encoded string) to a uuid.UUID object diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index 553f4e7a..d441d495 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -1,9 +1,10 @@ -import sys import json +import sys import time -from confluent_kafka import Producer, KafkaException -from urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse + +from confluent_kafka import KafkaException, Producer from .harvest_common import HarvestState, requests_retry_session diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py index 5e7702d9..45c2b8ea 100644 --- a/python/fatcat_tools/harvest/harvest_common.py +++ b/python/fatcat_tools/harvest/harvest_common.py @@ -1,14 +1,15 @@ -import sys -import json import datetime +import json +import sys + import requests +from confluent_kafka import Consumer, KafkaException, Producer, TopicPartition from requests.adapters import HTTPAdapter + # unclear why pylint chokes on this import. Recent 'requests' and 'urllib3' are # in Pipenv.lock, and there are no errors in QA -from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error -from confluent_kafka import Producer, Consumer, TopicPartition, KafkaException - +from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error # Used for parsing ISO date format (YYYY-MM-DD) DATE_FMT = "%Y-%m-%d" diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index c4e4a82a..0eb0343d 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -1,8 +1,9 @@ import sys import time + import sickle -from confluent_kafka import Producer, KafkaException +from confluent_kafka import KafkaException, Producer from .harvest_common import HarvestState diff --git a/python/fatcat_tools/importers/arabesque.py b/python/fatcat_tools/importers/arabesque.py index c8f7c77c..2b0ff7ec 100644 --- a/python/fatcat_tools/importers/arabesque.py +++ b/python/fatcat_tools/importers/arabesque.py @@ -1,7 +1,7 @@ import fatcat_openapi_client -from .common import EntityImporter, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS, b32_hex +from .common import SANE_MAX_RELEASES, SANE_MAX_URLS, EntityImporter, b32_hex, make_rel_url ARABESQUE_MATCH_WHERE_CLAUSE='WHERE hit = 1 AND identifier IS NOT NULL' diff --git a/python/fatcat_tools/importers/arxiv.py b/python/fatcat_tools/importers/arxiv.py index 43325ebc..fc429fb0 100644 --- a/python/fatcat_tools/importers/arxiv.py +++ b/python/fatcat_tools/importers/arxiv.py @@ -1,16 +1,16 @@ +import datetime +import json import re import sys -import json -import datetime + +import fatcat_openapi_client from bs4 import BeautifulSoup from pylatexenc.latex2text import LatexNodes2Text -import fatcat_openapi_client from .common import EntityImporter from .crossref import lookup_license_slug - latex2text = LatexNodes2Text() def latex_to_text(raw): diff --git a/python/fatcat_tools/importers/chocula.py b/python/fatcat_tools/importers/chocula.py index 5c9efe94..0b634e73 100644 --- a/python/fatcat_tools/importers/chocula.py +++ b/python/fatcat_tools/importers/chocula.py @@ -1,5 +1,6 @@ import fatcat_openapi_client + from .common import EntityImporter, clean diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 9d22ce83..e33a2012 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -1,32 +1,32 @@ -import re -import sys import csv +import datetime import json +import re import sqlite3 -import datetime import subprocess -from collections import Counter -from typing import Dict, Any, List, Optional, Tuple -import lxml +import sys import xml.etree.ElementTree as ET +from collections import Counter +from typing import Any, Dict, List, Optional, Tuple import elasticsearch +import fatcat_openapi_client +import fuzzycat.common +import fuzzycat.verify +import lxml from bs4 import BeautifulSoup from confluent_kafka import Consumer, KafkaException - -import fatcat_openapi_client from fatcat_openapi_client import ReleaseEntity from fatcat_openapi_client.rest import ApiException from fuzzycat.matching import match_release_fuzzy -import fuzzycat.common -import fuzzycat.verify # TODO: refactor so remove need for this (re-imports for backwards compatibility) -from fatcat_tools.normal import (clean_str as clean, is_cjk, b32_hex, LANG_MAP_MARC) # noqa: F401 +from fatcat_tools.normal import LANG_MAP_MARC, b32_hex +from fatcat_tools.normal import clean_str as clean # noqa: F401 +from fatcat_tools.normal import is_cjk from fatcat_tools.transforms import entity_to_dict - DATE_FMT: str = "%Y-%m-%d" SANE_MAX_RELEASES: int = 200 SANE_MAX_URLS: int = 100 diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py index 38c19a63..fd6936a4 100644 --- a/python/fatcat_tools/importers/crossref.py +++ b/python/fatcat_tools/importers/crossref.py @@ -1,14 +1,13 @@ -import sqlite3 import datetime -from typing import Dict, Optional, Any +import sqlite3 +from typing import Any, Dict, Optional import fatcat_openapi_client from fatcat_openapi_client import ReleaseEntity from .common import EntityImporter, clean - # The docs/guide should be the canonical home for these mappings; update there # first # Can get a list of Crossref types (with counts) via API: diff --git a/python/fatcat_tools/importers/datacite.py b/python/fatcat_tools/importers/datacite.py index 1593e6f8..a06c68a4 100644 --- a/python/fatcat_tools/importers/datacite.py +++ b/python/fatcat_tools/importers/datacite.py @@ -10,8 +10,8 @@ functions (parse_datacite_...), which may help testing. import collections import datetime -import re import json +import re import sqlite3 import sys diff --git a/python/fatcat_tools/importers/dblp_container.py b/python/fatcat_tools/importers/dblp_container.py index a9f993a8..3d280fb7 100644 --- a/python/fatcat_tools/importers/dblp_container.py +++ b/python/fatcat_tools/importers/dblp_container.py @@ -7,8 +7,9 @@ pre-scraped in to JSON from HTML pages. import sys # noqa: F401 import fatcat_openapi_client -from fatcat_tools.normal import clean_str + from fatcat_tools.importers.common import EntityImporter +from fatcat_tools.normal import clean_str class DblpContainerImporter(EntityImporter): diff --git a/python/fatcat_tools/importers/dblp_release.py b/python/fatcat_tools/importers/dblp_release.py index fa5cb842..6d028f2f 100644 --- a/python/fatcat_tools/importers/dblp_release.py +++ b/python/fatcat_tools/importers/dblp_release.py @@ -21,18 +21,25 @@ brittle/unreliable TSV lookup mechanism for prefix-to-container_id (as of December 2020). """ -import sys # noqa: F401 +import datetime import json +import sys # noqa: F401 import warnings -import datetime -from typing import List, Optional, Any +from typing import Any, List, Optional import fatcat_openapi_client -from fatcat_tools.normal import (clean_doi, clean_str, parse_month, - clean_orcid, clean_hdl, - clean_arxiv_id, clean_wikidata_qid, clean_isbn13) from fatcat_tools.importers.common import EntityImporter +from fatcat_tools.normal import ( + clean_arxiv_id, + clean_doi, + clean_hdl, + clean_isbn13, + clean_orcid, + clean_str, + clean_wikidata_qid, + parse_month, +) from fatcat_tools.transforms import entity_to_dict diff --git a/python/fatcat_tools/importers/doaj_article.py b/python/fatcat_tools/importers/doaj_article.py index 833089ae..1831c4cd 100644 --- a/python/fatcat_tools/importers/doaj_article.py +++ b/python/fatcat_tools/importers/doaj_article.py @@ -4,15 +4,24 @@ Importer for DOAJ article-level metadata, schema v1. DOAJ API schema and docs: https://doaj.org/api/v1/docs """ -import warnings import datetime +import warnings from typing import List, Optional import fatcat_openapi_client -from fatcat_tools.normal import (clean_doi, clean_str, parse_month, - clean_orcid, detect_text_lang, parse_lang_name, parse_country_name, - clean_pmid, clean_pmcid) + from fatcat_tools.importers.common import EntityImporter +from fatcat_tools.normal import ( + clean_doi, + clean_orcid, + clean_pmcid, + clean_pmid, + clean_str, + detect_text_lang, + parse_country_name, + parse_lang_name, + parse_month, +) # Cutoff length for abstracts. MAX_ABSTRACT_LENGTH = 2048 diff --git a/python/fatcat_tools/importers/file_meta.py b/python/fatcat_tools/importers/file_meta.py index 3d9f5923..0951ed84 100644 --- a/python/fatcat_tools/importers/file_meta.py +++ b/python/fatcat_tools/importers/file_meta.py @@ -1,5 +1,6 @@ import fatcat_openapi_client + from .common import EntityImporter diff --git a/python/fatcat_tools/importers/fileset_generic.py b/python/fatcat_tools/importers/fileset_generic.py index 13352fb2..43c2a49c 100644 --- a/python/fatcat_tools/importers/fileset_generic.py +++ b/python/fatcat_tools/importers/fileset_generic.py @@ -2,6 +2,7 @@ import fatcat_openapi_client from fatcat_tools import entity_from_dict + from .common import EntityImporter diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py index a811c856..0f666652 100644 --- a/python/fatcat_tools/importers/grobid_metadata.py +++ b/python/fatcat_tools/importers/grobid_metadata.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -import json import base64 +import json + import fatcat_openapi_client + from .common import EntityImporter, clean, make_rel_url MAX_ABSTRACT_BYTES=4096 diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py index 4d4efc0a..f0943c1e 100644 --- a/python/fatcat_tools/importers/ingest.py +++ b/python/fatcat_tools/importers/ingest.py @@ -2,6 +2,7 @@ import datetime import fatcat_openapi_client + from .common import EntityImporter, make_rel_url diff --git a/python/fatcat_tools/importers/jalc.py b/python/fatcat_tools/importers/jalc.py index 12f5450f..0a983c5e 100644 --- a/python/fatcat_tools/importers/jalc.py +++ b/python/fatcat_tools/importers/jalc.py @@ -1,12 +1,14 @@ -import sys -import sqlite3 import datetime -from bs4 import BeautifulSoup +import sqlite3 +import sys import fatcat_openapi_client +from bs4 import BeautifulSoup + from fatcat_tools.normal import clean_doi -from .common import EntityImporter, clean, is_cjk, DATE_FMT + +from .common import DATE_FMT, EntityImporter, clean, is_cjk def parse_jalc_persons(raw_persons): diff --git a/python/fatcat_tools/importers/journal_metadata.py b/python/fatcat_tools/importers/journal_metadata.py index 9f3b429f..25d7b3b5 100644 --- a/python/fatcat_tools/importers/journal_metadata.py +++ b/python/fatcat_tools/importers/journal_metadata.py @@ -1,5 +1,6 @@ import fatcat_openapi_client + from .common import EntityImporter, clean diff --git a/python/fatcat_tools/importers/jstor.py b/python/fatcat_tools/importers/jstor.py index 5d35f5e2..d37424d6 100644 --- a/python/fatcat_tools/importers/jstor.py +++ b/python/fatcat_tools/importers/jstor.py @@ -1,12 +1,13 @@ -import sys -import json import datetime +import json +import sys import warnings -from bs4 import BeautifulSoup import fatcat_openapi_client -from .common import EntityImporter, clean, LANG_MAP_MARC +from bs4 import BeautifulSoup + +from .common import LANG_MAP_MARC, EntityImporter, clean from .crossref import CONTAINER_TYPE_MAP # TODO: more entries? diff --git a/python/fatcat_tools/importers/orcid.py b/python/fatcat_tools/importers/orcid.py index 4412a46d..3bdd23a1 100644 --- a/python/fatcat_tools/importers/orcid.py +++ b/python/fatcat_tools/importers/orcid.py @@ -1,8 +1,11 @@ import sys + import fatcat_openapi_client + from .common import EntityImporter, clean + def value_or_none(e): if type(e) == dict: e = e.get('value') diff --git a/python/fatcat_tools/kafka.py b/python/fatcat_tools/kafka.py index 228de134..32749db2 100644 --- a/python/fatcat_tools/kafka.py +++ b/python/fatcat_tools/kafka.py @@ -1,5 +1,5 @@ -from confluent_kafka import Producer, KafkaException +from confluent_kafka import KafkaException, Producer def kafka_fail_fast(err, msg): diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py index 24c0bb0a..9b65e768 100644 --- a/python/fatcat_tools/normal.py +++ b/python/fatcat_tools/normal.py @@ -4,10 +4,10 @@ A bunch of helpers to parse and normalize strings: external identifiers, free-form input, titles, etc. """ -import re import base64 -from typing import Optional, Union +import re import unicodedata +from typing import Optional, Union import ftfy import langdetect diff --git a/python/fatcat_tools/references.py b/python/fatcat_tools/references.py index 3a2709a4..8361b260 100644 --- a/python/fatcat_tools/references.py +++ b/python/fatcat_tools/references.py @@ -5,18 +5,18 @@ index of reference links between works in the main catalog. See bulk citation and citation API proposals for design documentation. """ -import sys -import datetime import argparse -from typing import Optional, List, Any, Dict, Union +import datetime +import sys +from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, validator import elasticsearch from elasticsearch_dsl import Search from fatcat_openapi_client import ReleaseEntity +from pydantic import BaseModel, validator from fatcat_tools import public_api -from fatcat_tools.transforms.access import release_access_options, AccessOption +from fatcat_tools.transforms.access import AccessOption, release_access_options from fatcat_tools.transforms.entities import entity_to_dict diff --git a/python/fatcat_tools/reviewers/review_common.py b/python/fatcat_tools/reviewers/review_common.py index b4930c19..867d826d 100644 --- a/python/fatcat_tools/reviewers/review_common.py +++ b/python/fatcat_tools/reviewers/review_common.py @@ -1,9 +1,9 @@ -import time import datetime import subprocess +import time from collections import Counter -from typing import Optional, List, Any +from typing import Any, List, Optional import fatcat_openapi_client diff --git a/python/fatcat_tools/transforms/access.py b/python/fatcat_tools/transforms/access.py index 39d4c6d3..ae9880e7 100644 --- a/python/fatcat_tools/transforms/access.py +++ b/python/fatcat_tools/transforms/access.py @@ -1,9 +1,9 @@ from enum import Enum -from typing import Optional, List +from typing import List, Optional -from pydantic import BaseModel from fatcat_openapi_client import ReleaseEntity +from pydantic import BaseModel class AccessType(str, Enum): diff --git a/python/fatcat_tools/transforms/csl.py b/python/fatcat_tools/transforms/csl.py index 0556f4fe..f8b26bce 100644 --- a/python/fatcat_tools/transforms/csl.py +++ b/python/fatcat_tools/transforms/csl.py @@ -1,9 +1,13 @@ import json -from citeproc import CitationStylesStyle, CitationStylesBibliography -from citeproc import Citation, CitationItem -from citeproc import formatter +from citeproc import ( + Citation, + CitationItem, + CitationStylesBibliography, + CitationStylesStyle, + formatter, +) from citeproc.source.json import CiteProcJSON from citeproc_styles import get_style_filepath diff --git a/python/fatcat_tools/transforms/elasticsearch.py b/python/fatcat_tools/transforms/elasticsearch.py index ec5891c3..1826d4eb 100644 --- a/python/fatcat_tools/transforms/elasticsearch.py +++ b/python/fatcat_tools/transforms/elasticsearch.py @@ -1,10 +1,15 @@ import datetime -from typing import Dict, Any, Optional +from typing import Any, Dict, Optional import tldextract - -from fatcat_openapi_client import ReleaseEntity, ContainerEntity, EntityEdit, ChangelogEntry, FileEntity +from fatcat_openapi_client import ( + ChangelogEntry, + ContainerEntity, + EntityEdit, + FileEntity, + ReleaseEntity, +) def check_kbart(year: int, archive: dict) -> Optional[bool]: diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index 982ee3ea..a61e364c 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -1,7 +1,8 @@ import json import time -from confluent_kafka import Consumer, Producer, KafkaException + +from confluent_kafka import Consumer, KafkaException, Producer from fatcat_tools.transforms import release_ingest_request, release_to_elasticsearch diff --git a/python/fatcat_util.py b/python/fatcat_util.py index a45b2ba4..a8e99ac3 100755 --- a/python/fatcat_util.py +++ b/python/fatcat_util.py @@ -7,10 +7,10 @@ TODO: - create entity from JSON (?) """ -import sys import argparse +import sys -from fatcat_tools import uuid2fcid, fcid2uuid, authenticated_api +from fatcat_tools import authenticated_api, fcid2uuid, uuid2fcid def run_uuid2fcid(args): diff --git a/python/fatcat_web/__init__.py b/python/fatcat_web/__init__.py index 3207bc75..336b4133 100644 --- a/python/fatcat_web/__init__.py +++ b/python/fatcat_web/__init__.py @@ -1,23 +1,22 @@ import sys +import elasticsearch +import fatcat_openapi_client +from authlib.flask.client import OAuth from flask import Flask from flask.logging import create_logger -from flask_uuid import FlaskUUID from flask_debugtoolbar import DebugToolbarExtension from flask_login import LoginManager -from flask_wtf.csrf import CSRFProtect from flask_misaka import Misaka from flask_mwoauth import MWOAuth -from authlib.flask.client import OAuth -from loginpass import create_flask_blueprint, Gitlab, GitHub, ORCiD +from flask_uuid import FlaskUUID +from flask_wtf.csrf import CSRFProtect +from loginpass import GitHub, Gitlab, ORCiD, create_flask_blueprint from raven.contrib.flask import Sentry -import fatcat_openapi_client -import elasticsearch from fatcat_web.web_config import Config - toolbar = DebugToolbarExtension() app = Flask(__name__, static_url_path='/static') app.config.from_object(Config) @@ -76,7 +75,7 @@ app.register_blueprint(mwoauth.bp, url_prefix='/auth/wikipedia') app.es_client = elasticsearch.Elasticsearch(Config.ELASTICSEARCH_BACKEND) -from fatcat_web import routes, editing_routes, ref_routes, auth, cors, forms +from fatcat_web import auth, cors, editing_routes, forms, ref_routes, routes # TODO: blocking on ORCID support in loginpass if Config.ORCID_CLIENT_ID: diff --git a/python/fatcat_web/auth.py b/python/fatcat_web/auth.py index 73869544..137bc2bb 100644 --- a/python/fatcat_web/auth.py +++ b/python/fatcat_web/auth.py @@ -1,11 +1,14 @@ from collections import namedtuple -import requests -import pymacaroons -from flask import render_template, abort, redirect, session, flash -from flask_login import logout_user, login_user, UserMixin -from fatcat_web import login_manager, app, api, priv_api, Config + import fatcat_openapi_client +import pymacaroons +import requests +from flask import abort, flash, redirect, render_template, session +from flask_login import UserMixin, login_user, logout_user + +from fatcat_web import Config, api, app, login_manager, priv_api + def handle_logout(): logout_user() diff --git a/python/fatcat_web/cors.py b/python/fatcat_web/cors.py index d84babd8..cb2054b2 100644 --- a/python/fatcat_web/cors.py +++ b/python/fatcat_web/cors.py @@ -6,7 +6,8 @@ This snippet from: http://flask.pocoo.org/snippets/56/ from datetime import timedelta from functools import update_wrapper -from flask import make_response, request, current_app + +from flask import current_app, make_response, request def crossdomain(origin=None, methods=None, headers=None, diff --git a/python/fatcat_web/graphics.py b/python/fatcat_web/graphics.py index 0765a777..b5a83f6c 100644 --- a/python/fatcat_web/graphics.py +++ b/python/fatcat_web/graphics.py @@ -1,9 +1,10 @@ -from typing import List, Tuple, Dict +from typing import Dict, List, Tuple import pygal from pygal.style import CleanStyle + def ia_coverage_histogram(rows: List[Tuple]) -> pygal.Graph: """ Note: this returns a raw pygal chart; it does not render it to SVG/PNG diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 95f1f5c9..73781016 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -4,17 +4,18 @@ Helpers for doing elasticsearch queries (used in the web interface; not part of the formal API) """ -import sys import datetime +import sys from dataclasses import dataclass -from typing import List, Optional, Any +from typing import Any, List, Optional import elasticsearch -from elasticsearch_dsl import Search, Q import elasticsearch_dsl.response +from elasticsearch_dsl import Q, Search from fatcat_web import app + class FatcatSearchError(Exception): def __init__(self, status_code: int, name: str, description: str = None): diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py index b0a50e0f..c15fefa4 100644 --- a/python/fatcat_web/web_config.py +++ b/python/fatcat_web/web_config.py @@ -10,9 +10,10 @@ import scripts. """ import os -import raven import subprocess +import raven + basedir = os.path.abspath(os.path.dirname(__file__)) class Config(object): diff --git a/python/fatcat_webface.py b/python/fatcat_webface.py index 94dce9ca..d12e8dad 100755 --- a/python/fatcat_webface.py +++ b/python/fatcat_webface.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse + from fatcat_web import app + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) diff --git a/python/fatcat_worker.py b/python/fatcat_worker.py index c3efb5f4..397cf731 100755 --- a/python/fatcat_worker.py +++ b/python/fatcat_worker.py @@ -1,11 +1,18 @@ #!/usr/bin/env python3 -import sys import argparse +import sys + import raven from fatcat_tools import public_api -from fatcat_tools.workers import ChangelogWorker, EntityUpdatesWorker, ElasticsearchReleaseWorker, ElasticsearchContainerWorker, ElasticsearchChangelogWorker +from fatcat_tools.workers import ( + ChangelogWorker, + ElasticsearchChangelogWorker, + ElasticsearchContainerWorker, + ElasticsearchReleaseWorker, + EntityUpdatesWorker, +) # Yep, a global. Gets DSN from `SENTRY_DSN` environment variable sentry_client = raven.Client() diff --git a/python/shell.py b/python/shell.py index d53911b9..9b561f5f 100644 --- a/python/shell.py +++ b/python/shell.py @@ -1,14 +1,15 @@ # flake8: noqa # bunch of libraries one might want -import uuid -import datetime -import requests import argparse +import datetime +import uuid import fatcat_openapi_client +import requests from fatcat_openapi_client import * from fatcat_openapi_client.rest import ApiException + from fatcat_tools import * if __name__ == '__main__': diff --git a/python/tests/api_editgroups.py b/python/tests/api_editgroups.py index b4420b29..9f870fb1 100644 --- a/python/tests/api_editgroups.py +++ b/python/tests/api_editgroups.py @@ -1,7 +1,7 @@ -import pytest import datetime +import pytest from fatcat_openapi_client import * from fixtures import * diff --git a/python/tests/api_editor.py b/python/tests/api_editor.py index ab118501..0c680a2f 100644 --- a/python/tests/api_editor.py +++ b/python/tests/api_editor.py @@ -1,8 +1,6 @@ -import pytest - import fatcat_openapi_client - +import pytest from fixtures import api diff --git a/python/tests/api_entity_editing.py b/python/tests/api_entity_editing.py index fee4e34f..5f22e9f2 100644 --- a/python/tests/api_entity_editing.py +++ b/python/tests/api_entity_editing.py @@ -1,6 +1,5 @@ import pytest - from fatcat_openapi_client import * from fixtures import * diff --git a/python/tests/api_entity_state.py b/python/tests/api_entity_state.py index 4b49083e..58e1d42b 100644 --- a/python/tests/api_entity_state.py +++ b/python/tests/api_entity_state.py @@ -1,6 +1,5 @@ import pytest - from fatcat_openapi_client import * from fatcat_openapi_client.rest import ApiException from fixtures import * diff --git a/python/tests/api_releases.py b/python/tests/api_releases.py index 4437b57b..a823d4cd 100644 --- a/python/tests/api_releases.py +++ b/python/tests/api_releases.py @@ -1,7 +1,7 @@ -import pytest import datetime +import pytest from fatcat_openapi_client import * from fixtures import * diff --git a/python/tests/api_webcaptures.py b/python/tests/api_webcaptures.py index 85813218..510d398c 100644 --- a/python/tests/api_webcaptures.py +++ b/python/tests/api_webcaptures.py @@ -1,7 +1,7 @@ -import pytest import datetime +import pytest from fatcat_openapi_client import * from fixtures import * diff --git a/python/tests/clean_files.py b/python/tests/clean_files.py index f752bc2c..c8b935bb 100644 --- a/python/tests/clean_files.py +++ b/python/tests/clean_files.py @@ -1,11 +1,12 @@ import copy -import pytest -from fatcat_tools.cleanups import FileCleaner +import pytest from fatcat_openapi_client import * from fixtures import * +from fatcat_tools.cleanups import FileCleaner + @pytest.fixture(scope="function") def file_cleaner(api): diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py index e0bb1aae..a9b81bda 100644 --- a/python/tests/fixtures.py +++ b/python/tests/fixtures.py @@ -1,14 +1,13 @@ +import elasticsearch +import fatcat_openapi_client import pytest from dotenv import load_dotenv -import elasticsearch +from fatcat_openapi_client import * import fatcat_web -import fatcat_openapi_client -from fatcat_openapi_client import * from fatcat_tools import authenticated_api - ES_CONTAINER_STATS_RESP = { 'timed_out': False, 'aggregations': { diff --git a/python/tests/harvest_crossref.py b/python/tests/harvest_crossref.py index cad0f03b..fb43e3f6 100644 --- a/python/tests/harvest_crossref.py +++ b/python/tests/harvest_crossref.py @@ -1,7 +1,9 @@ -import json import datetime +import json + import responses + from fatcat_tools.harvest import * diff --git a/python/tests/harvest_datacite.py b/python/tests/harvest_datacite.py index 13c6042a..0a7584bd 100644 --- a/python/tests/harvest_datacite.py +++ b/python/tests/harvest_datacite.py @@ -1,7 +1,9 @@ -import json import datetime +import json + import responses + from fatcat_tools.harvest import * diff --git a/python/tests/harvest_pubmed.py b/python/tests/harvest_pubmed.py index 58bc4226..aba766ba 100644 --- a/python/tests/harvest_pubmed.py +++ b/python/tests/harvest_pubmed.py @@ -2,8 +2,9 @@ Test pubmed FTP harvest. """ -import os import datetime +import os + import pytest from fatcat_tools.harvest import * diff --git a/python/tests/harvest_state.py b/python/tests/harvest_state.py index cc624d97..ecd276ed 100644 --- a/python/tests/harvest_state.py +++ b/python/tests/harvest_state.py @@ -1,5 +1,6 @@ import datetime + from fatcat_tools.harvest import * diff --git a/python/tests/import_arabesque.py b/python/tests/import_arabesque.py index 20cde3a6..6f73d817 100644 --- a/python/tests/import_arabesque.py +++ b/python/tests/import_arabesque.py @@ -1,10 +1,11 @@ import json -import pytest -from fatcat_tools.importers import ArabesqueMatchImporter, SqlitePusher, JsonLinePusher +import pytest from fixtures import * +from fatcat_tools.importers import ArabesqueMatchImporter, JsonLinePusher, SqlitePusher + @pytest.fixture(scope="function") def arabesque_importer(api): diff --git a/python/tests/import_arxiv.py b/python/tests/import_arxiv.py index 3ed1ab29..9368fb06 100644 --- a/python/tests/import_arxiv.py +++ b/python/tests/import_arxiv.py @@ -1,9 +1,9 @@ import pytest from bs4 import BeautifulSoup +from fixtures import * from fatcat_tools.importers import ArxivRawImporter, Bs4XmlFilePusher -from fixtures import * @pytest.fixture(scope="function") diff --git a/python/tests/import_common.py b/python/tests/import_common.py index 69e51432..2b7c935a 100644 --- a/python/tests/import_common.py +++ b/python/tests/import_common.py @@ -1,17 +1,17 @@ -import json import datetime +import json from typing import Any -import pytest import elasticsearch import fatcat_openapi_client -from fatcat_openapi_client import ReleaseEntity, ReleaseExtIds import fuzzycat.matching +import pytest +from fatcat_openapi_client import ReleaseEntity, ReleaseExtIds +from fixtures import * from fatcat_tools.importers import EntityImporter from fatcat_tools.transforms import entity_to_dict -from fixtures import * @pytest.fixture(scope="function") diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py index a2c8681b..a8a1ec44 100644 --- a/python/tests/import_crossref.py +++ b/python/tests/import_crossref.py @@ -1,12 +1,12 @@ -import json import gzip +import json + import pytest +from fixtures import api from fatcat_tools.importers import CrossrefImporter, JsonLinePusher -from fixtures import api - @pytest.fixture(scope="function") def crossref_importer(api): diff --git a/python/tests/import_datacite.py b/python/tests/import_datacite.py index a92a732d..e1c79bc4 100644 --- a/python/tests/import_datacite.py +++ b/python/tests/import_datacite.py @@ -2,27 +2,26 @@ Test datacite importer. """ +import collections +import datetime import gzip import json -import datetime -import collections +import fatcat_openapi_client import pytest +from fixtures import api from fatcat_tools.importers import DataciteImporter, JsonLinePusher from fatcat_tools.importers.datacite import ( - find_original_language_title, - parse_datacite_titles, - parse_datacite_dates, clean_doi, + contributor_list_contains_contributor, + find_original_language_title, index_form_to_display_name, lookup_license_slug, - contributor_list_contains_contributor, + parse_datacite_dates, + parse_datacite_titles, ) from fatcat_tools.transforms import entity_to_dict -import fatcat_openapi_client -from fixtures import api -import json @pytest.fixture(scope="function") diff --git a/python/tests/import_dblp.py b/python/tests/import_dblp.py index 13a52247..4fb178b8 100644 --- a/python/tests/import_dblp.py +++ b/python/tests/import_dblp.py @@ -1,11 +1,17 @@ import io + import pytest from bs4 import BeautifulSoup - -from fatcat_tools.importers import DblpReleaseImporter, DblpContainerImporter, Bs4XmlLargeFilePusher, JsonLinePusher from fixtures import * +from fatcat_tools.importers import ( + Bs4XmlLargeFilePusher, + DblpContainerImporter, + DblpReleaseImporter, + JsonLinePusher, +) + @pytest.fixture(scope="function") def dblp_importer(api): diff --git a/python/tests/import_doaj.py b/python/tests/import_doaj.py index 72a3acb8..4cd3b518 100644 --- a/python/tests/import_doaj.py +++ b/python/tests/import_doaj.py @@ -1,14 +1,14 @@ -import json import datetime +import json -import pytest import elasticsearch import fatcat_openapi_client +import pytest +from fixtures import * from fatcat_tools.importers import DoajArticleImporter, JsonLinePusher from fatcat_tools.transforms import entity_to_dict -from fixtures import * @pytest.fixture(scope="function") diff --git a/python/tests/import_file_generic.py b/python/tests/import_file_generic.py index cef82777..e1d987e8 100644 --- a/python/tests/import_file_generic.py +++ b/python/tests/import_file_generic.py @@ -1,8 +1,8 @@ import pytest +from fatcat_openapi_client import * from fatcat_tools.importers.common import EntityImporter -from fatcat_openapi_client import * def test_file_update_generic(): diff --git a/python/tests/import_file_meta.py b/python/tests/import_file_meta.py index e6d1a59f..949a1075 100644 --- a/python/tests/import_file_meta.py +++ b/python/tests/import_file_meta.py @@ -1,11 +1,12 @@ import json -import pytest -from fatcat_tools.importers import FileMetaImporter, JsonLinePusher +import pytest from fatcat_openapi_client import FileEntity from fixtures import * +from fatcat_tools.importers import FileMetaImporter, JsonLinePusher + @pytest.fixture(scope="function") def file_meta_importer(api): diff --git a/python/tests/import_fileset_generic.py b/python/tests/import_fileset_generic.py index 44310304..8d30a83b 100644 --- a/python/tests/import_fileset_generic.py +++ b/python/tests/import_fileset_generic.py @@ -1,9 +1,10 @@ import json + import pytest +from fixtures import * from fatcat_tools.importers import FilesetImporter, JsonLinePusher -from fixtures import * @pytest.fixture(scope="function") diff --git a/python/tests/import_grobid_metadata.py b/python/tests/import_grobid_metadata.py index 87cb8ef0..862b310f 100644 --- a/python/tests/import_grobid_metadata.py +++ b/python/tests/import_grobid_metadata.py @@ -1,11 +1,12 @@ -import os -import json import base64 +import json +import os + import pytest +from fixtures import * from fatcat_tools.importers import GrobidMetadataImporter, LinePusher -from fixtures import * """ WARNING: these tests are currently very fragile because they have database diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py index 955c97cb..04a8e7f6 100644 --- a/python/tests/import_ingest.py +++ b/python/tests/import_ingest.py @@ -1,10 +1,15 @@ import json -import pytest -from fatcat_tools.importers import IngestFileResultImporter, IngestWebResultImporter, JsonLinePusher +import pytest from fixtures import * +from fatcat_tools.importers import ( + IngestFileResultImporter, + IngestWebResultImporter, + JsonLinePusher, +) + @pytest.fixture(scope="function") def ingest_importer(api): diff --git a/python/tests/import_jalc.py b/python/tests/import_jalc.py index 010dfce8..c11e2307 100644 --- a/python/tests/import_jalc.py +++ b/python/tests/import_jalc.py @@ -1,10 +1,10 @@ import pytest from bs4 import BeautifulSoup - -from fatcat_tools.importers import JalcImporter, Bs4XmlFilePusher, Bs4XmlLinesPusher from fixtures import * +from fatcat_tools.importers import Bs4XmlFilePusher, Bs4XmlLinesPusher, JalcImporter + @pytest.fixture(scope="function") def jalc_importer(api): diff --git a/python/tests/import_journal_metadata.py b/python/tests/import_journal_metadata.py index 51b0a78a..cdf4e8ed 100644 --- a/python/tests/import_journal_metadata.py +++ b/python/tests/import_journal_metadata.py @@ -1,8 +1,8 @@ import pytest +from fixtures import * from fatcat_tools.importers import JournalMetadataImporter, JsonLinePusher -from fixtures import * @pytest.fixture(scope="function") diff --git a/python/tests/import_jstor.py b/python/tests/import_jstor.py index 25a904a0..270da3ec 100644 --- a/python/tests/import_jstor.py +++ b/python/tests/import_jstor.py @@ -1,10 +1,10 @@ import pytest from bs4 import BeautifulSoup - -from fatcat_tools.importers import JstorImporter, Bs4XmlFilePusher from fixtures import * +from fatcat_tools.importers import Bs4XmlFilePusher, JstorImporter + @pytest.fixture(scope="function") def jstor_importer(api): diff --git a/python/tests/import_matched.py b/python/tests/import_matched.py index 6b61c53c..aad8e799 100644 --- a/python/tests/import_matched.py +++ b/python/tests/import_matched.py @@ -1,10 +1,11 @@ import json -import pytest -from fatcat_tools.importers import MatchedImporter, JsonLinePusher +import pytest from fixtures import * +from fatcat_tools.importers import JsonLinePusher, MatchedImporter + @pytest.fixture(scope="function") def matched_importer(api): diff --git a/python/tests/import_orcid.py b/python/tests/import_orcid.py index f78ccde7..88a4b871 100644 --- a/python/tests/import_orcid.py +++ b/python/tests/import_orcid.py @@ -1,10 +1,11 @@ import json -import pytest -from fatcat_tools.importers import OrcidImporter, JsonLinePusher +import pytest from fixtures import * +from fatcat_tools.importers import JsonLinePusher, OrcidImporter + @pytest.fixture(scope="function") def orcid_importer(api): diff --git a/python/tests/import_pubmed.py b/python/tests/import_pubmed.py index 8b5ff3e5..371c59bb 100644 --- a/python/tests/import_pubmed.py +++ b/python/tests/import_pubmed.py @@ -1,10 +1,10 @@ import pytest from bs4 import BeautifulSoup - -from fatcat_tools.importers import PubmedImporter, Bs4XmlLargeFilePusher from fixtures import * +from fatcat_tools.importers import Bs4XmlLargeFilePusher, PubmedImporter + @pytest.fixture(scope="function") def pubmed_importer(api): diff --git a/python/tests/import_shadow.py b/python/tests/import_shadow.py index 40a1d589..1caaf780 100644 --- a/python/tests/import_shadow.py +++ b/python/tests/import_shadow.py @@ -1,10 +1,11 @@ import json -import pytest -from fatcat_tools.importers import ShadowLibraryImporter, JsonLinePusher +import pytest from fixtures import * +from fatcat_tools.importers import JsonLinePusher, ShadowLibraryImporter + @pytest.fixture(scope="function") def shadow_importer(api): diff --git a/python/tests/importer.py b/python/tests/importer.py index 40bd8cba..1f0e942f 100644 --- a/python/tests/importer.py +++ b/python/tests/importer.py @@ -1,7 +1,8 @@ -from fatcat_tools.importers import CrossrefImporter, OrcidImporter from fixtures import * +from fatcat_tools.importers import CrossrefImporter, OrcidImporter + def test_issnl_mapping_lookup(api): with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: diff --git a/python/tests/tools_api.py b/python/tests/tools_api.py index a4b5f2ea..6d514210 100644 --- a/python/tests/tools_api.py +++ b/python/tests/tools_api.py @@ -2,7 +2,7 @@ import pytest from fatcat_openapi_client.rest import ApiException -from fatcat_tools import public_api, authenticated_api +from fatcat_tools import authenticated_api, public_api def test_authenticated_api(): diff --git a/python/tests/transform_ingest.py b/python/tests/transform_ingest.py index a61aa4a7..6a358c8e 100644 --- a/python/tests/transform_ingest.py +++ b/python/tests/transform_ingest.py @@ -1,11 +1,12 @@ import json -from fatcat_tools.transforms import release_ingest_request from fatcat_openapi_client import * from fixtures import api from import_crossref import crossref_importer +from fatcat_tools.transforms import release_ingest_request + def test_basic_ingest_release(crossref_importer): with open('tests/files/crossref-works.single.json', 'r') as f: diff --git a/python/tests/web_citation_csl.py b/python/tests/web_citation_csl.py index a72742cb..508600e7 100644 --- a/python/tests/web_citation_csl.py +++ b/python/tests/web_citation_csl.py @@ -1,5 +1,6 @@ import json + import pytest from fixtures import * diff --git a/python/tests/web_coverage.py b/python/tests/web_coverage.py index 47123686..0dbf3df8 100644 --- a/python/tests/web_coverage.py +++ b/python/tests/web_coverage.py @@ -1,8 +1,8 @@ -import json -import pytest import datetime +import json +import pytest from fatcat_openapi_client.rest import ApiException from fixtures import * diff --git a/python/tests/web_entity_views.py b/python/tests/web_entity_views.py index 45fd3e9a..0d51f87e 100644 --- a/python/tests/web_entity_views.py +++ b/python/tests/web_entity_views.py @@ -1,8 +1,9 @@ import json + from fixtures import * -from fatcat_web.forms import ReleaseEntityForm, FileEntityForm, ContainerEntityForm +from fatcat_web.forms import ContainerEntityForm, FileEntityForm, ReleaseEntityForm DUMMY_DEMO_ENTITIES = { 'container': ('aaaaaaaaaaaaaeiraaaaaaaaai', '00000000-0000-0000-1111-fff000000002'), diff --git a/python/tests/web_refs.py b/python/tests/web_refs.py index 72a295e1..0ab69607 100644 --- a/python/tests/web_refs.py +++ b/python/tests/web_refs.py @@ -1,11 +1,12 @@ import json -import pytest -from fatcat_web.search import get_elastic_container_random_releases +import pytest from fatcat_openapi_client.rest import ApiException from fixtures import * +from fatcat_web.search import get_elastic_container_random_releases + def test_basic_refs(app, mocker): diff --git a/python/tests/web_search.py b/python/tests/web_search.py index 8df01466..fdcffdea 100644 --- a/python/tests/web_search.py +++ b/python/tests/web_search.py @@ -1,11 +1,12 @@ import json -import pytest -from fatcat_web.search import get_elastic_container_random_releases +import pytest from fatcat_openapi_client.rest import ApiException from fixtures import * +from fatcat_web.search import get_elastic_container_random_releases + def test_generic_search(app): |