summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fatcat_scholar/issue_db.py2
-rw-r--r--fatcat_scholar/query_citation.py2
-rw-r--r--fatcat_scholar/query_fatcat.py4
-rw-r--r--fatcat_scholar/search.py4
-rw-r--r--fatcat_scholar/templates/search_macros.html2
-rw-r--r--fatcat_scholar/worker.py4
-rw-r--r--settings.toml41
-rw-r--r--tests/test_issue_db.py2
8 files changed, 35 insertions, 26 deletions
diff --git a/fatcat_scholar/issue_db.py b/fatcat_scholar/issue_db.py
index bf79b09..e01c6d9 100644
--- a/fatcat_scholar/issue_db.py
+++ b/fatcat_scholar/issue_db.py
@@ -426,7 +426,7 @@ def main() -> None:
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = settings.FATCAT_API_HOST
api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(api_conf))
- es_client = elasticsearch.Elasticsearch(settings.FATCAT_ELASTICSEARCH_BACKEND)
+ es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_FATCAT_BASE)
if args.func == "load_pubs":
idb.load_pubs(args.json_file, api)
diff --git a/fatcat_scholar/query_citation.py b/fatcat_scholar/query_citation.py
index c555fd4..21f4a9c 100644
--- a/fatcat_scholar/query_citation.py
+++ b/fatcat_scholar/query_citation.py
@@ -188,7 +188,7 @@ if __name__ == "__main__":
release = ref_to_release(ref)
print(release)
- es_backend = os.environ.get("ELASTICSEARCH_BACKEND", "https://search.fatcat.wiki")
+ es_backend = os.environ.get("ELASTICSEARCH_FATCAT_BASE", "https://search.fatcat.wiki")
es_client = elasticsearch.Elasticsearch(es_backend)
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = os.environ.get("FATCAT_API_HOST", "https://api.fatcat.wiki/v0")
diff --git a/fatcat_scholar/query_fatcat.py b/fatcat_scholar/query_fatcat.py
index c862bc0..0f5347b 100644
--- a/fatcat_scholar/query_fatcat.py
+++ b/fatcat_scholar/query_fatcat.py
@@ -44,8 +44,8 @@ def run_query_fatcat(query: str, fulltext_only: bool, json_output: Any) -> None:
"""
api_session = requests_retry_session()
- es_backend = os.environ.get("ELASTICSEARCH_BACKEND", "https://search.fatcat.wiki")
- es_index = "fatcat_release"
+ es_backend = os.environ.get("ELASTICSEARCH_FATCAT_BASE", "https://search.fatcat.wiki")
+ es_index = os.environ.get("ELASTICSEARCH_FATCAT_RELEASE_INDEX", "fatcat_release")
es_client = elasticsearch.Elasticsearch(es_backend)
search = Search(using=es_client, index=es_index)
diff --git a/fatcat_scholar/search.py b/fatcat_scholar/search.py
index 600745f..c75ccdb 100644
--- a/fatcat_scholar/search.py
+++ b/fatcat_scholar/search.py
@@ -97,7 +97,7 @@ class FulltextHits(BaseModel):
# global sync client connection
-es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_BACKEND, timeout=25.0)
+es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_QUERY_BASE, timeout=25.0)
def transform_es_results(resp: Response) -> List[dict]:
@@ -281,7 +281,7 @@ def do_fulltext_search(
query: FulltextQuery, deep_page_limit: int = 2000
) -> FulltextHits:
- search = Search(using=es_client, index=settings.ELASTICSEARCH_FULLTEXT_INDEX)
+ search = Search(using=es_client, index=settings.ELASTICSEARCH_QUERY_FULLTEXT_INDEX)
if query.collapse_key:
search = search.filter("term", collapse_key=query.collapse_key)
diff --git a/fatcat_scholar/templates/search_macros.html b/fatcat_scholar/templates/search_macros.html
index 6c7cb56..0f43142 100644
--- a/fatcat_scholar/templates/search_macros.html
+++ b/fatcat_scholar/templates/search_macros.html
@@ -392,7 +392,7 @@
{# TODO: remove doc for ES 7.x-style lack of type #}
{# TODO: only show 'json' link if from cluster? #}
{% if debug_mode %}
- <a target="_blank" rel="noopener" href="{{ settings.ELASTICSEARCH_FRONTEND }}/{{ settings.ELASTICSEARCH_FULLTEXT_INDEX }}/_doc/{{ paper.key }}">
+ <a target="_blank" rel="noopener" href="{{ settings.ELASTICSEARCH_PUBLIC_URL }}/{{ settings.ELASTICSEARCH_QUERY_FULLTEXT_INDEX }}/_doc/{{ paper.key }}">
<span class="ui label small" title="search document JSON debug link">json</span>
</a>
{% endif %}
diff --git a/fatcat_scholar/worker.py b/fatcat_scholar/worker.py
index 78b54d4..fb806a7 100644
--- a/fatcat_scholar/worker.py
+++ b/fatcat_scholar/worker.py
@@ -227,7 +227,7 @@ def main() -> None:
fdw.run()
elif args.worker == "index-docs-worker":
es_client = elasticsearch.Elasticsearch(
- settings.ELASTICSEARCH_BACKEND, timeout=25.0
+ settings.ELASTICSEARCH_WRITE_BASE, timeout=25.0
)
idw = IndexDocsWorker(
kafka_brokers=settings.KAFKA_BROKERS,
@@ -235,7 +235,7 @@ def main() -> None:
consume_topics=[f"scholar-{settings.SCHOLAR_ENV}.update-docs"],
consumer_group=f"scholar-{settings.SCHOLAR_ENV}-index-workers",
es_client=es_client,
- es_index=settings.ELASTICSEARCH_FULLTEXT_INDEX,
+ es_index=settings.ELASTICSEARCH_WRITE_FULLTEXT_INDEX,
)
idw.run()
diff --git a/settings.toml b/settings.toml
index b2faf21..efdc322 100644
--- a/settings.toml
+++ b/settings.toml
@@ -3,16 +3,19 @@
SCHOLAR_ENV = "default"
SCHOLAR_ISSUEDB_PATH = "data/issue_db.sqlite"
I18N_LANG_DEFAULT = "en"
-FATCAT_ELASTICSEARCH_BACKEND = "https://search.fatcat.wiki"
-ELASTICSEARCH_BACKEND = "http://localhost:9200"
-ELASTICSEARCH_FRONTEND = "http://search-svc097.fatcat.wiki:9292"
-ELASTICSEARCH_FULLTEXT_INDEX = "scholar_fulltext"
+ELASTICSEARCH_QUERY_BASE = "http://localhost:9200"
+ELASTICSEARCH_QUERY_FULLTEXT_INDEX = "scholar_fulltext"
+ELASTICSEARCH_WRITE_BASE = "http://localhost:9200"
+ELASTICSEARCH_WRITE_FULLTEXT_INDEX = "scholar_fulltext_v01"
+ELASTICSEARCH_PUBLIC_URL = "http://localhost:9292"
THUMBNAIL_URL_PREFIX = "https://blobs.fatcat.wiki/thumbnail/pdf/"
SANDCRAWLER_DB_API = "http://aitio.us.archive.org:3030"
SANDCRAWLER_S3_API = "wbgrp-svc169.us.archive.org:8333"
GROBID_HOST = "https://grobid.qa.fatcat.wiki"
KAFKA_BROKERS = []
FATCAT_API_HOST = "https://api.fatcat.wiki/v0"
+ELASTICSEARCH_FATCAT_BASE = "https://search.fatcat.wiki"
+ELASTICSEARCH_FATCAT_RELEASE_INDEX = "fatcat_release"
INDEX_WORKER_BATCH_SIZE = 50
SENTRY_DSN = ""
ENABLE_GOATCOUNTER = false
@@ -27,9 +30,11 @@ EXCLUDE_WEB_PUBLISHERS = []
[test]
SCHOLAR_ENV = "test"
FATCAT_API_HOST = "http://disabled-during-tests-bogus.xyz:3333"
-FATCAT_ELASTICSEARCH_BACKEND = "http://disabled-during-tests-bogus.xyz:3333"
-ELASTICSEARCH_FRONTEND = "http://disabled-during-tests-bogus.xyz:3333"
-ELASTICSEARCH_BACKEND = "http://disabled-during-tests-bogus.xyz:3333"
+ELASTICSEARCH_FATCAT_BASE = "http://disabled-during-tests-bogus.xyz:3333"
+ELASTICSEARCH_QUERY_BASE = "http://disabled-during-tests-bogus.xyz:3333"
+ELASTICSEARCH_WRITE_BASE = "http://disabled-during-tests-bogus.xyz:3333"
+ELASTICSEARCH_WRITE_FULLTEXT_INDEX = "dummy_index"
+ELASTICSEARCH_PUBLIC_URL = "http://disabled-during-tests-bogus.xyz:3333"
SANDCRAWLER_DB_API = "http://disabled-during-tests-bogus.xyz:3333"
SANDCRAWLER_S3_API = "disabled-during-tests-bogus.xyz:8333"
GROBID_HOST = "http://disabled-during-tests-bogus.xyz:3333"
@@ -37,23 +42,27 @@ SCHOLAR_ISSUEDB_PATH = ":memory:"
[development]
SCHOLAR_ENV = "dev"
-ELASTICSEARCH_FULLTEXT_INDEX = "dev_scholar_fulltext"
-ELASTICSEARCH_BACKEND = "http://localhost:9200"
-ELASTICSEARCH_FRONTEND = "http://localhost:9200"
+ELASTICSEARCH_QUERY_BASE = "http://localhost:9200"
+ELASTICSEARCH_QUERY_FULLTEXT_INDEX = "dev_scholar_fulltext"
+ELASTICSEARCH_WRITE_BASE = "http://localhost:9200"
+ELASTICSEARCH_WRITE_FULLTEXT_INDEX = "dev_scholar_fulltext"
+ELASTICSEARCH_PUBLIC_URL = "http://localhost:9200"
KAFKA_BROKERS = ["localhost"]
-FATCAT_API_HOST = "http://localhost:9411/v0"
[qa]
SCHOLAR_ENV = "qa"
-ELASTICSEARCH_FRONTEND = "http://search-svc097.fatcat.wiki:9292"
-ELASTICSEARCH_FULLTEXT_INDEX = "scholar_fulltext_v01"
+ELASTICSEARCH_QUERY_BASE = "http://scholar-svc500.fatcat.wiki:9292"
+ELASTICSEARCH_WRITE_BASE = "http://localhost:9200"
+ELASTICSEARCH_PUBLIC_URL = "http://scholar-svc500.fatcat.wiki:9292"
KAFKA_BROKERS = ["wbgrp-svc263.us.archive.org"]
[prod]
SCHOLAR_ENV = "prod"
-FATCAT_ELASTICSEARCH_BACKEND = "https://search.qa.fatcat.wiki"
-ELASTICSEARCH_FULLTEXT_INDEX = "scholar_fulltext_v01"
-ELASTICSEARCH_BACKEND = "http://localhost:9200"
+ELASTICSEARCH_QUERY_BASE = "http://scholar-svc500.fatcat.wiki:9292"
+ELASTICSEARCH_QUERY_FULLTEXT_INDEX = "scholar_fulltext"
+ELASTICSEARCH_WRITE_BASE = "http://localhost:9200"
+ELASTICSEARCH_WRITE_FULLTEXT_INDEX = "scholar_fulltext_v01"
+ELASTICSEARCH_PUBLIC_URL = "http://scholar-svc500.fatcat.wiki:9292"
KAFKA_BROKERS = ["wbgrp-svc263.us.archive.org"]
ENABLE_GOATCOUNTER = true
GOATCOUNTER_ENDPOINT = "/goatcounter/count"
diff --git a/tests/test_issue_db.py b/tests/test_issue_db.py
index 965b925..9fdab94 100644
--- a/tests/test_issue_db.py
+++ b/tests/test_issue_db.py
@@ -11,7 +11,7 @@ def test_issue_db_basics() -> None:
api_conf.host = settings.FATCAT_API_HOST
api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(api_conf))
- es_client = elasticsearch.Elasticsearch(settings.FATCAT_ELASTICSEARCH_BACKEND)
+ es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_FATCAT_BASE)
issue_db = IssueDB(settings.SCHOLAR_ISSUEDB_PATH)
issue_db.init_db()
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-22 18:25:11 +0000