summaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/fatcat_web/graphics.py42
-rw-r--r--python/fatcat_web/routes.py20
-rw-r--r--python/fatcat_web/search.py22
-rw-r--r--python/fatcat_web/templates/entity_macros.html12
-rw-r--r--python/fatcat_web/web_config.py3
5 files changed, 80 insertions, 19 deletions
diff --git a/python/fatcat_web/graphics.py b/python/fatcat_web/graphics.py
index 7d6e5702..a8e217e3 100644
--- a/python/fatcat_web/graphics.py
+++ b/python/fatcat_web/graphics.py
@@ -37,14 +37,17 @@ def ia_coverage_histogram(rows: List[Tuple]) -> pygal.Graph:
chart.add('Missing', [y['missing'] for y in years])
return chart
-def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
"""
Note: this returns a raw pygal chart; it does not render it to SVG/PNG
"""
years = sorted(rows, key=lambda x: x['year'])
- CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+ if merge_shadows:
+ CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+ else:
+ CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
label_count = len(years)
if len(years) > 30:
label_count = 10
@@ -55,20 +58,26 @@ def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph:
chart.x_title = "Year"
#chart.y_title = "Count"
chart.x_labels = [str(y['year']) for y in years]
- chart.add('None', [y['none'] for y in years])
- chart.add('Shadow', [y['shadows_only'] for y in years])
+ if merge_shadows:
+ chart.add('None', [y['none'] + y['shadows_only'] for y in years])
+ else:
+ chart.add('None', [y['none'] for y in years])
+ chart.add('Shadow', [y['shadows_only'] for y in years])
chart.add('Dark', [y['dark'] for y in years])
chart.add('Bright', [y['bright'] for y in years])
return chart
-def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
"""
Note: this returns a raw pygal chart; it does not render it to SVG/PNG
"""
dates = sorted(rows, key=lambda x: x['date'])
- CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+ if merge_shadows:
+ CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+ else:
+ CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
label_count = len(dates)
if len(dates) > 30:
label_count = 10
@@ -79,20 +88,26 @@ def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph:
chart.x_title = "Date"
#chart.y_title = "Count"
chart.x_labels = [str(y['date']) for y in dates]
- chart.add('None', [y['none'] for y in dates])
- chart.add('Shadow', [y['shadows_only'] for y in dates])
+ if merge_shadows:
+ chart.add('None', [y['none'] + y['shadows_only'] for y in dates])
+ else:
+ chart.add('None', [y['none'] for y in dates])
+ chart.add('Shadow', [y['shadows_only'] for y in dates])
chart.add('Dark', [y['dark'] for y in dates])
chart.add('Bright', [y['bright'] for y in dates])
return chart
-def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
"""
Note: this returns a raw pygal chart; it does not render it to SVG/PNG
"""
volumes = sorted(rows, key=lambda x: x['volume'])
- CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+ if merge_shadows:
+ CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+ else:
+ CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
label_count = len(volumes)
if len(volumes) >= 30:
label_count = 10
@@ -103,8 +118,11 @@ def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph:
chart.x_title = "Volume"
#chart.y_title = "Count"
chart.x_labels = [str(y['volume']) for y in volumes]
- chart.add('None', [y['none'] for y in volumes])
- chart.add('Shadow', [y['shadows_only'] for y in volumes])
+ if merge_shadows:
+ chart.add('None', [y['none'] + y['shadows_only'] for y in volumes])
+ else:
+ chart.add('None', [y['none'] for y in volumes])
+ chart.add('Shadow', [y['shadows_only'] for y in volumes])
chart.add('Dark', [y['dark'] for y in volumes])
chart.add('Bright', [y['bright'] for y in volumes])
return chart
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index 1281a617..bcfa133b 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -754,10 +754,16 @@ def coverage_search():
coverage_type_preservation = get_elastic_preservation_by_type(query)
if query.recent:
date_histogram = get_elastic_preservation_by_date(query)
- date_histogram_svg = preservation_by_date_histogram(date_histogram).render_data_uri()
+ date_histogram_svg = preservation_by_date_histogram(
+ date_histogram,
+ merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+ ).render_data_uri()
else:
year_histogram = get_elastic_preservation_by_year(query)
- year_histogram_svg = preservation_by_year_histogram(year_histogram).render_data_uri()
+ year_histogram_svg = preservation_by_year_histogram(
+ year_histogram,
+ merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+ ).render_data_uri()
return render_template(
'coverage_search.html',
query=query,
@@ -886,7 +892,10 @@ def container_ident_preservation_by_year_svg(ident):
except Exception as ae:
app.log.error(ae)
abort(503)
- return preservation_by_year_histogram(histogram).render_response()
+ return preservation_by_year_histogram(
+ histogram,
+ merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+ ).render_response()
@app.route('/container/<ident>/preservation_by_volume.json', methods=['GET', 'OPTIONS'])
@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type'])
@@ -914,7 +923,10 @@ def container_ident_preservation_by_volume_svg(ident):
except Exception as ae:
app.log.error(ae)
abort(503)
- return preservation_by_volume_histogram(histogram).render_response()
+ return preservation_by_volume_histogram(
+ histogram,
+ merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+ ).render_response()
@app.route('/release/<ident>.bib', methods=['GET'])
def release_bibtex(ident):
diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py
index b0d27b2e..7d9ce69f 100644
--- a/python/fatcat_web/search.py
+++ b/python/fatcat_web/search.py
@@ -402,6 +402,9 @@ def get_elastic_search_coverage(query: ReleaseQuery) -> dict:
for k in ('bright', 'dark', 'shadows_only', 'none'):
if not k in preservation_bucket:
preservation_bucket[k] = 0
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ preservation_bucket['none'] += preservation_bucket['shadows_only']
+ preservation_bucket['shadows_only'] = 0
stats = {
'total': resp.hits.total,
'preservation': preservation_bucket,
@@ -464,6 +467,9 @@ def get_elastic_container_stats(ident, issnl=None):
for k in ('bright', 'dark', 'shadows_only', 'none'):
if not k in preservation_bucket:
preservation_bucket[k] = 0
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ preservation_bucket['none'] += preservation_bucket['shadows_only']
+ preservation_bucket['shadows_only'] = 0
release_type_bucket = agg_to_dict(resp.aggregations.release_type)
stats = {
'ident': ident,
@@ -602,6 +608,10 @@ def get_elastic_preservation_by_year(query) -> List[dict]:
year_dicts[num] = dict(year=num, bright=0, dark=0, shadows_only=0, none=0)
for row in buckets:
year_dicts[int(row['key']['year'])][row['key']['preservation']] = int(row['doc_count'])
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ for k in year_dicts.keys():
+ year_dicts[k]['none'] += year_dicts[k]['shadows_only']
+ year_dicts[k]['shadows_only'] = 0
return sorted(year_dicts.values(), key=lambda x: x['year'])
@@ -675,6 +685,10 @@ def get_elastic_preservation_by_date(query) -> List[dict]:
this_date = this_date + datetime.timedelta(days=1)
for row in buckets:
date_dicts[row['key']['date'][0:10]][row['key']['preservation']] = int(row['doc_count'])
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ for k in date_dicts.keys():
+ date_dicts[k]['none'] += date_dicts[k]['shadows_only']
+ date_dicts[k]['shadows_only'] = 0
return sorted(date_dicts.values(), key=lambda x: x['date'])
def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict]:
@@ -728,6 +742,10 @@ def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict
for row in buckets:
if row['key']['volume'].isdigit():
volume_dicts[int(row['key']['volume'])][row['key']['preservation']] = int(row['doc_count'])
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ for k in volume_dicts.keys():
+ volume_dicts[k]['none'] += volume_dicts[k]['shadows_only']
+ volume_dicts[k]['shadows_only'] = 0
return sorted(volume_dicts.values(), key=lambda x: x['volume'])
def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:
@@ -797,4 +815,8 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:
for k in type_set:
for p in ('bright', 'dark', 'shadows_only', 'none'):
type_dicts[k]['total'] += type_dicts[k][p]
+ if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+ for k in type_set:
+ type_dicts[k]['none'] += type_dicts[k]['shadows_only']
+ type_dicts[k]['shadows_only'] = 0
return sorted(type_dicts.values(), key=lambda x: x['total'], reverse=True)
diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html
index c3ae099a..718c071c 100644
--- a/python/fatcat_web/templates/entity_macros.html
+++ b/python/fatcat_web/templates/entity_macros.html
@@ -265,7 +265,6 @@ yellow
{% set frac_bright = stats.bright/stats.total %}
{% set frac_dark = stats.dark/stats.total %}
- {% set frac_shadows_only = stats.shadows_only/stats.total %}
{% set frac_none = stats.none/stats.total %}
<div class="ui {{ extra_class }} multiple progress" data-percent="0,0,0,0" style="margin-bottom: 0.1em;">
@@ -275,9 +274,12 @@ yellow
<div class="green bar" style="border-radius: 0; min-width: 0; width: {{ (frac_dark*100)|round(method='ceil') }}%; background-color: darkgreen;" title="dark">
<div class="progress">{# {{ (frac_dark*100)|int }}% #}</div>
</div>
+ {% if stats.shadows_only %}
+ {% set frac_shadows_only = stats.shadows_only/stats.total %}
<div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_shadows_only*100)|round(method='ceil') }}%; background-color: darkred;" title="shadows only">
<div class="progress">{# {{ (frac_shadows_only*100)|int }}% #}</div>
</div>
+ {% endif %}
<div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_none*100)|round(method='ceil') }}%;" title="no preservation">
<div class="progress">{# {{ (frac_none*100)|int }}% #}</div>
</div>
@@ -290,7 +292,6 @@ yellow
{% set frac_bright = stats.bright/stats.total %}
{% set frac_dark = stats.dark/stats.total %}
- {% set frac_shadows_only = stats.shadows_only/stats.total %}
{% set frac_none = stats.none/stats.total %}
<table class="ui very basic very compact collapsing table" style="font-weight: bold; margin-left: 1em;">
@@ -305,11 +306,14 @@ yellow
<td class="right aligned" >{{ "{:,}".format(stats.dark) }}
<td class="right aligned" >{{ (frac_dark*100)|round(2,method='ceil') }}%
<td>preserved but not publicly accessible (dark)
+ {% if stats.shadows_only %}
+ {% set frac_shadows_only = stats.shadows_only/stats.total %}
<tr>
<td style="background-color: darkred;">
<td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }}
<td class="right aligned" >{{ (frac_shadows_only*100)|round(2,method='ceil') }}%
<td>only independently preserved in "shadow" libraries
+ {% endif %}
<tr>
<td style="background-color: #db2828;">
<td class="right aligned" >{{ "{:,}".format(stats.none) }}
@@ -324,7 +328,6 @@ yellow
{% set frac_bright = stats.bright/stats.total %}
{% set frac_dark = stats.dark/stats.total %}
- {% set frac_shadows_only = stats.shadows_only/stats.total %}
{% set frac_none = stats.none/stats.total %}
<table class="ui very basic very compact collapsing table">
@@ -337,10 +340,13 @@ yellow
<td style="background-color: darkgreen;">
<td class="right aligned" >{{ "{:,}".format(stats.dark) }}
<td>preserved, inaccessible (dark)
+ {% if stats.shadows_only %}
+ {% set frac_shadows_only = stats.shadows_only/stats.total %}
<tr>
<td style="background-color: darkred;">
<td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }}
<td>shadow library only
+ {% endif %}
<tr>
<td style="background-color: #db2828;">
<td class="right aligned" >{{ "{:,}".format(stats.none) }}
diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py
index 344f1c2a..22a704d9 100644
--- a/python/fatcat_web/web_config.py
+++ b/python/fatcat_web/web_config.py
@@ -52,6 +52,9 @@ class Config(object):
IA_XAUTH_CLIENT_ID = os.environ.get("IA_XAUTH_CLIENT_ID", default=None)
IA_XAUTH_CLIENT_SECRET = os.environ.get("IA_XAUTH_CLIENT_SECRET", default=None)
+ # controls granularity of "shadow_only" preservation category
+ FATCAT_MERGE_SHADOW_PRESERVATION = os.environ.get("FATCAT_MERGE_SHADOW_PRESERVATION", default=False)
+
# CSRF on by default, but only for WTF forms (not, eg, search, lookups, GET
# forms)
WTF_CSRF_CHECK_DEFAULT = False
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-23 19:02:44 +0000