diff options
| -rw-r--r-- | python/fatcat_web/graphics.py | 42 | ||||
| -rw-r--r-- | python/fatcat_web/routes.py | 20 | ||||
| -rw-r--r-- | python/fatcat_web/search.py | 22 | ||||
| -rw-r--r-- | python/fatcat_web/templates/entity_macros.html | 12 | ||||
| -rw-r--r-- | python/fatcat_web/web_config.py | 3 |
5 files changed, 80 insertions, 19 deletions
diff --git a/python/fatcat_web/graphics.py b/python/fatcat_web/graphics.py index 7d6e5702..a8e217e3 100644 --- a/python/fatcat_web/graphics.py +++ b/python/fatcat_web/graphics.py @@ -37,14 +37,17 @@ def ia_coverage_histogram(rows: List[Tuple]) -> pygal.Graph: chart.add('Missing', [y['missing'] for y in years]) return chart -def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph: +def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: """ Note: this returns a raw pygal chart; it does not render it to SVG/PNG """ years = sorted(rows, key=lambda x: x['year']) - CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") + if merge_shadows: + CleanStyle.colors = ("red", "darkolivegreen", "limegreen") + else: + CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") label_count = len(years) if len(years) > 30: label_count = 10 @@ -55,20 +58,26 @@ def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph: chart.x_title = "Year" #chart.y_title = "Count" chart.x_labels = [str(y['year']) for y in years] - chart.add('None', [y['none'] for y in years]) - chart.add('Shadow', [y['shadows_only'] for y in years]) + if merge_shadows: + chart.add('None', [y['none'] + y['shadows_only'] for y in years]) + else: + chart.add('None', [y['none'] for y in years]) + chart.add('Shadow', [y['shadows_only'] for y in years]) chart.add('Dark', [y['dark'] for y in years]) chart.add('Bright', [y['bright'] for y in years]) return chart -def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph: +def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: """ Note: this returns a raw pygal chart; it does not render it to SVG/PNG """ dates = sorted(rows, key=lambda x: x['date']) - CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") + if merge_shadows: + CleanStyle.colors = ("red", "darkolivegreen", "limegreen") + else: + CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") label_count = len(dates) if len(dates) > 30: label_count = 10 @@ -79,20 +88,26 @@ def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph: chart.x_title = "Date" #chart.y_title = "Count" chart.x_labels = [str(y['date']) for y in dates] - chart.add('None', [y['none'] for y in dates]) - chart.add('Shadow', [y['shadows_only'] for y in dates]) + if merge_shadows: + chart.add('None', [y['none'] + y['shadows_only'] for y in dates]) + else: + chart.add('None', [y['none'] for y in dates]) + chart.add('Shadow', [y['shadows_only'] for y in dates]) chart.add('Dark', [y['dark'] for y in dates]) chart.add('Bright', [y['bright'] for y in dates]) return chart -def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph: +def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: """ Note: this returns a raw pygal chart; it does not render it to SVG/PNG """ volumes = sorted(rows, key=lambda x: x['volume']) - CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") + if merge_shadows: + CleanStyle.colors = ("red", "darkolivegreen", "limegreen") + else: + CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen") label_count = len(volumes) if len(volumes) >= 30: label_count = 10 @@ -103,8 +118,11 @@ def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph: chart.x_title = "Volume" #chart.y_title = "Count" chart.x_labels = [str(y['volume']) for y in volumes] - chart.add('None', [y['none'] for y in volumes]) - chart.add('Shadow', [y['shadows_only'] for y in volumes]) + if merge_shadows: + chart.add('None', [y['none'] + y['shadows_only'] for y in volumes]) + else: + chart.add('None', [y['none'] for y in volumes]) + chart.add('Shadow', [y['shadows_only'] for y in volumes]) chart.add('Dark', [y['dark'] for y in volumes]) chart.add('Bright', [y['bright'] for y in volumes]) return chart diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 1281a617..bcfa133b 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -754,10 +754,16 @@ def coverage_search(): coverage_type_preservation = get_elastic_preservation_by_type(query) if query.recent: date_histogram = get_elastic_preservation_by_date(query) - date_histogram_svg = preservation_by_date_histogram(date_histogram).render_data_uri() + date_histogram_svg = preservation_by_date_histogram( + date_histogram, + merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION, + ).render_data_uri() else: year_histogram = get_elastic_preservation_by_year(query) - year_histogram_svg = preservation_by_year_histogram(year_histogram).render_data_uri() + year_histogram_svg = preservation_by_year_histogram( + year_histogram, + merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION, + ).render_data_uri() return render_template( 'coverage_search.html', query=query, @@ -886,7 +892,10 @@ def container_ident_preservation_by_year_svg(ident): except Exception as ae: app.log.error(ae) abort(503) - return preservation_by_year_histogram(histogram).render_response() + return preservation_by_year_histogram( + histogram, + merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION, + ).render_response() @app.route('/container/<ident>/preservation_by_volume.json', methods=['GET', 'OPTIONS']) @crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) @@ -914,7 +923,10 @@ def container_ident_preservation_by_volume_svg(ident): except Exception as ae: app.log.error(ae) abort(503) - return preservation_by_volume_histogram(histogram).render_response() + return preservation_by_volume_histogram( + histogram, + merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION, + ).render_response() @app.route('/release/<ident>.bib', methods=['GET']) def release_bibtex(ident): diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index b0d27b2e..7d9ce69f 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -402,6 +402,9 @@ def get_elastic_search_coverage(query: ReleaseQuery) -> dict: for k in ('bright', 'dark', 'shadows_only', 'none'): if not k in preservation_bucket: preservation_bucket[k] = 0 + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + preservation_bucket['none'] += preservation_bucket['shadows_only'] + preservation_bucket['shadows_only'] = 0 stats = { 'total': resp.hits.total, 'preservation': preservation_bucket, @@ -464,6 +467,9 @@ def get_elastic_container_stats(ident, issnl=None): for k in ('bright', 'dark', 'shadows_only', 'none'): if not k in preservation_bucket: preservation_bucket[k] = 0 + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + preservation_bucket['none'] += preservation_bucket['shadows_only'] + preservation_bucket['shadows_only'] = 0 release_type_bucket = agg_to_dict(resp.aggregations.release_type) stats = { 'ident': ident, @@ -602,6 +608,10 @@ def get_elastic_preservation_by_year(query) -> List[dict]: year_dicts[num] = dict(year=num, bright=0, dark=0, shadows_only=0, none=0) for row in buckets: year_dicts[int(row['key']['year'])][row['key']['preservation']] = int(row['doc_count']) + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + for k in year_dicts.keys(): + year_dicts[k]['none'] += year_dicts[k]['shadows_only'] + year_dicts[k]['shadows_only'] = 0 return sorted(year_dicts.values(), key=lambda x: x['year']) @@ -675,6 +685,10 @@ def get_elastic_preservation_by_date(query) -> List[dict]: this_date = this_date + datetime.timedelta(days=1) for row in buckets: date_dicts[row['key']['date'][0:10]][row['key']['preservation']] = int(row['doc_count']) + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + for k in date_dicts.keys(): + date_dicts[k]['none'] += date_dicts[k]['shadows_only'] + date_dicts[k]['shadows_only'] = 0 return sorted(date_dicts.values(), key=lambda x: x['date']) def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict]: @@ -728,6 +742,10 @@ def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict for row in buckets: if row['key']['volume'].isdigit(): volume_dicts[int(row['key']['volume'])][row['key']['preservation']] = int(row['doc_count']) + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + for k in volume_dicts.keys(): + volume_dicts[k]['none'] += volume_dicts[k]['shadows_only'] + volume_dicts[k]['shadows_only'] = 0 return sorted(volume_dicts.values(), key=lambda x: x['volume']) def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]: @@ -797,4 +815,8 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]: for k in type_set: for p in ('bright', 'dark', 'shadows_only', 'none'): type_dicts[k]['total'] += type_dicts[k][p] + if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: + for k in type_set: + type_dicts[k]['none'] += type_dicts[k]['shadows_only'] + type_dicts[k]['shadows_only'] = 0 return sorted(type_dicts.values(), key=lambda x: x['total'], reverse=True) diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index c3ae099a..718c071c 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -265,7 +265,6 @@ yellow {% set frac_bright = stats.bright/stats.total %} {% set frac_dark = stats.dark/stats.total %} - {% set frac_shadows_only = stats.shadows_only/stats.total %} {% set frac_none = stats.none/stats.total %} <div class="ui {{ extra_class }} multiple progress" data-percent="0,0,0,0" style="margin-bottom: 0.1em;"> @@ -275,9 +274,12 @@ yellow <div class="green bar" style="border-radius: 0; min-width: 0; width: {{ (frac_dark*100)|round(method='ceil') }}%; background-color: darkgreen;" title="dark"> <div class="progress">{# {{ (frac_dark*100)|int }}% #}</div> </div> + {% if stats.shadows_only %} + {% set frac_shadows_only = stats.shadows_only/stats.total %} <div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_shadows_only*100)|round(method='ceil') }}%; background-color: darkred;" title="shadows only"> <div class="progress">{# {{ (frac_shadows_only*100)|int }}% #}</div> </div> + {% endif %} <div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_none*100)|round(method='ceil') }}%;" title="no preservation"> <div class="progress">{# {{ (frac_none*100)|int }}% #}</div> </div> @@ -290,7 +292,6 @@ yellow {% set frac_bright = stats.bright/stats.total %} {% set frac_dark = stats.dark/stats.total %} - {% set frac_shadows_only = stats.shadows_only/stats.total %} {% set frac_none = stats.none/stats.total %} <table class="ui very basic very compact collapsing table" style="font-weight: bold; margin-left: 1em;"> @@ -305,11 +306,14 @@ yellow <td class="right aligned" >{{ "{:,}".format(stats.dark) }} <td class="right aligned" >{{ (frac_dark*100)|round(2,method='ceil') }}% <td>preserved but not publicly accessible (dark) + {% if stats.shadows_only %} + {% set frac_shadows_only = stats.shadows_only/stats.total %} <tr> <td style="background-color: darkred;"> <td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }} <td class="right aligned" >{{ (frac_shadows_only*100)|round(2,method='ceil') }}% <td>only independently preserved in "shadow" libraries + {% endif %} <tr> <td style="background-color: #db2828;"> <td class="right aligned" >{{ "{:,}".format(stats.none) }} @@ -324,7 +328,6 @@ yellow {% set frac_bright = stats.bright/stats.total %} {% set frac_dark = stats.dark/stats.total %} - {% set frac_shadows_only = stats.shadows_only/stats.total %} {% set frac_none = stats.none/stats.total %} <table class="ui very basic very compact collapsing table"> @@ -337,10 +340,13 @@ yellow <td style="background-color: darkgreen;"> <td class="right aligned" >{{ "{:,}".format(stats.dark) }} <td>preserved, inaccessible (dark) + {% if stats.shadows_only %} + {% set frac_shadows_only = stats.shadows_only/stats.total %} <tr> <td style="background-color: darkred;"> <td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }} <td>shadow library only + {% endif %} <tr> <td style="background-color: #db2828;"> <td class="right aligned" >{{ "{:,}".format(stats.none) }} diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py index 344f1c2a..22a704d9 100644 --- a/python/fatcat_web/web_config.py +++ b/python/fatcat_web/web_config.py @@ -52,6 +52,9 @@ class Config(object): IA_XAUTH_CLIENT_ID = os.environ.get("IA_XAUTH_CLIENT_ID", default=None) IA_XAUTH_CLIENT_SECRET = os.environ.get("IA_XAUTH_CLIENT_SECRET", default=None) + # controls granularity of "shadow_only" preservation category + FATCAT_MERGE_SHADOW_PRESERVATION = os.environ.get("FATCAT_MERGE_SHADOW_PRESERVATION", default=False) + # CSRF on by default, but only for WTF forms (not, eg, search, lookups, GET # forms) WTF_CSRF_CHECK_DEFAULT = False |